From 4b1151017d3b30d706b813552cfddd80dc4bc7b2 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 22 Aug 2017 11:41:24 +0200 Subject: [PATCH 001/723] TCP: fix offer issue --- src/net/tcp/tcp.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index 528003e2a0..4ba46bc5ab 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -489,11 +489,12 @@ void TCP::request_offer(Connection& conn) { debug2(" %s requestin offer: uw=%u rem=%u\n", conn.to_string().c_str(), conn.usable_window(), conn.sendq_remaining()); - if (packets > 0) { - conn.offer(packets); - } + // Note: Must be called even if packets is 0 + // because the connectoin is responsible for requeuing itself (see Connection::offer) + conn.offer(packets); } + void TCP::queue_offer(Connection& conn) { if(not conn.is_queued() and conn.can_send()) From 127c02991ca71ae721bdf1d6b73eede8f0bc5bf2 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 12 Oct 2017 15:35:57 +0200 Subject: [PATCH 002/723] tcp sack: more test cases etc. WIP --- api/net/tcp/sack.hpp | 11 ++-- test/net/unit/tcp_sack_test.cpp | 101 ++++++++++++++++++++++++-------- 2 files changed, 82 insertions(+), 30 deletions(-) diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index adbff5ddbe..73346b03e7 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -94,7 +94,7 @@ using Entries = std::array; struct Ack_result { Entries entries; - uint32_t bytes_freed; + uint32_t bytes; }; @@ -102,7 +102,7 @@ template class List { public: - Entries recv_out_of_order(seq_t seq, size_t len) + Ack_result recv_out_of_order(seq_t seq, size_t len) { return impl.recv_out_of_order(seq, len); } @@ -121,7 +121,7 @@ class Array_list { public: static_assert(N <= 32 && N > 0, "N wrong sized - optimized for small N"); - Entries recv_out_of_order(seq_t seq, uint32_t len) + Ack_result recv_out_of_order(seq_t seq, uint32_t len) { Block inc{seq, seq+len}; @@ -174,15 +174,14 @@ class Array_list { } else { update = get_free(); if (not update) { - // TODO: return older sack list - std::cout << "No free blocks left \n"; + return {recent_entries(), 0}; } else { *update = inc; } } update_latest(update); - return recent_entries(); + return {recent_entries(), len}; } Ack_result new_valid_ack(seq_t seq) diff --git a/test/net/unit/tcp_sack_test.cpp b/test/net/unit/tcp_sack_test.cpp index 7bea48cd27..a83c2356da 100644 --- a/test/net/unit/tcp_sack_test.cpp +++ b/test/net/unit/tcp_sack_test.cpp @@ -62,35 +62,35 @@ CASE("SACK List Array implementation [RFC 2018]") 8500 5000 5500 9000 */ List> sack_list; - Entries entries; + Ack_result res; // 5000 5500 6000 - entries = sack_list.recv_out_of_order(5500, 500); - EXPECT(entries == expected({5500,6000})); + res = sack_list.recv_out_of_order(5500, 500); + EXPECT(res.entries == expected({5500,6000})); // 5000 5500 6500 - entries = sack_list.recv_out_of_order(6000, 500); - EXPECT(entries == expected({5500,6500})); + res = sack_list.recv_out_of_order(6000, 500); + EXPECT(res.entries == expected({5500,6500})); // 5000 5500 7000 - entries = sack_list.recv_out_of_order(6500, 500); - EXPECT(entries == expected({5500,7000})); + res = sack_list.recv_out_of_order(6500, 500); + EXPECT(res.entries == expected({5500,7000})); // 5000 5500 7500 - entries = sack_list.recv_out_of_order(7000,500); - EXPECT(entries == expected({5500,7500})); + res = sack_list.recv_out_of_order(7000,500); + EXPECT(res.entries == expected({5500,7500})); // 5000 5500 8000 - entries = sack_list.recv_out_of_order(7500,500); - EXPECT(entries == expected({5500,8000})); + res = sack_list.recv_out_of_order(7500,500); + EXPECT(res.entries == expected({5500,8000})); // 5000 5500 8500 - entries = sack_list.recv_out_of_order(8000,500); - EXPECT(entries == expected({5500,8500})); + res = sack_list.recv_out_of_order(8000,500); + EXPECT(res.entries == expected({5500,8500})); // 5000 5500 9000 - entries = sack_list.recv_out_of_order(8500,500); - EXPECT(entries == expected({5500, 9000})); + res = sack_list.recv_out_of_order(8500,500); + EXPECT(res.entries == expected({5500, 9000})); /* @@ -117,16 +117,16 @@ CASE("SACK List Array implementation [RFC 2018]") sack_list = List>(); // 5500 6000 6500 - entries = sack_list.recv_out_of_order(6000, 500); - EXPECT(entries == expected({6000,6500})); + res = sack_list.recv_out_of_order(6000, 500); + EXPECT(res.entries == expected({6000,6500})); // 5500 7000 7500 6000 6500 - entries = sack_list.recv_out_of_order(7000, 500); - EXPECT(entries == expected({7000,7500}, {6000,6500})); + res = sack_list.recv_out_of_order(7000, 500); + EXPECT(res.entries == expected({7000,7500}, {6000,6500})); // 5500 8000 8500 7000 7500 6000 6500 - entries = sack_list.recv_out_of_order(8000, 500); - EXPECT(entries == expected({8000,8500}, {7000,7500}, {6000,6500})); + res = sack_list.recv_out_of_order(8000, 500); + EXPECT(res.entries == expected({8000,8500}, {7000,7500}, {6000,6500})); /* Suppose at this point, the 4th packet is received out of order. @@ -146,12 +146,12 @@ CASE("SACK List Array implementation [RFC 2018]") for(auto& b : sack_list.impl.blocks) std::cout << b << "\n"; // 6000 7500 8000 8500 - entries = sack_list.recv_out_of_order(6500, 500); + res = sack_list.recv_out_of_order(6500, 500); for(auto& b : sack_list.impl.blocks) std::cout << b << "\n"; - EXPECT(entries == expected({6000,7500}, {8000,8500})); + EXPECT(res.entries == expected({6000,7500}, {8000,8500})); /* Suppose at this point, the 2nd segment is received. The data @@ -167,7 +167,60 @@ CASE("SACK List Array implementation [RFC 2018]") for(auto& b : sack_list.impl.blocks) std::cout << b << "\n"; EXPECT(result.entries == expected({8000,8500})); - EXPECT(result.bytes_freed == 7500-6000); + EXPECT(result.bytes == 7500-6000); } +CASE("SACK block list is full") +{ + + using namespace net::tcp::sack; + + constexpr int list_size = 9; + List> sack_list; + Ack_result res; + + + uint32_t seq = 1000; + int incr = 1000; + int blksz = 500; + for (int i = 0; i < list_size; i++){ + + // Fill with a 100b hole from previous entry + res = sack_list.recv_out_of_order(seq, blksz); + seq += incr; + } + + // Fully populated sack list + EXPECT(res.entries == expected({seq - incr, (seq - incr ) + blksz}, + {seq - incr * 2, (seq - incr * 2) + blksz }, + {seq - incr * 3, (seq - incr * 3) + blksz } )); + + std::cout << "Fully populated SACK list: \n" + << res.entries << "\n"; + + EXPECT(res.bytes == blksz); + + // Try adding one more + res = sack_list.recv_out_of_order(seq, blksz); + + // We should now get the same + EXPECT(res.entries == expected({seq - incr, (seq - incr ) + blksz}, + {seq - incr * 2, (seq - incr * 2) + blksz }, + {seq - incr * 3, (seq - incr * 3) + blksz } )); + + EXPECT(res.bytes == 0); + + // Add a block that connects to the end, which should free up one spot + res = sack_list.recv_out_of_order(seq - incr + blksz, blksz); + EXPECT(res.bytes == blksz); + + // Last block should now be larger + EXPECT(res.entries == expected({seq - incr, seq + blksz}, + {seq - incr * 2, (seq - incr * 2) + blksz }, + {seq - incr * 3, (seq - incr * 3) + blksz } )); + + + // Add a block that connects two blocks, which should free up one spot + +} From 03d40c3996908ce66d4e0b2be80aa1648206a6e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 18 Oct 2017 09:57:44 +0200 Subject: [PATCH 003/723] util: Added custom allocator for a fixed size list --- api/util/fixed_list_alloc.hpp | 106 ++++++++++++++++++ api/util/fixed_storage.hpp | 134 +++++++++++++++++++++++ test/util/unit/fixed_list_alloc_test.cpp | 61 +++++++++++ 3 files changed, 301 insertions(+) create mode 100644 api/util/fixed_list_alloc.hpp create mode 100644 api/util/fixed_storage.hpp create mode 100644 test/util/unit/fixed_list_alloc_test.cpp diff --git a/api/util/fixed_list_alloc.hpp b/api/util/fixed_list_alloc.hpp new file mode 100644 index 0000000000..8b748f8dc4 --- /dev/null +++ b/api/util/fixed_list_alloc.hpp @@ -0,0 +1,106 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef UTIL_FIXED_LIST_ALLOC_HPP +#define UTIL_FIXED_LIST_ALLOC_HPP + +#include "fixed_storage.hpp" + +// Implemenation based upon Howard Hinnant's terrific short_alloc example +// https://howardhinnant.github.io/short_alloc.h + +/** + * @brief A fixed list allocator. Keeps its own fixed storage. + * + * @note Only tested together with list, + * should not be used with other containers. + * + * @tparam T The type of object to be allocated + * @tparam N The maximum number of objects in storage + * @tparam Align The alignment of the object + */ +template +class Fixed_list_alloc { +public: + using value_type = T; + static auto constexpr alignment = Align; + static auto constexpr size = N; + using storage_type = Fixed_storage; + + static_assert((sizeof(T) * size) % alignment == 0, + "Total size (sizeof(T) * N) needs to be a multiple of alignment Align"); + +private: + storage_type store_; + +public: + Fixed_list_alloc() noexcept + : store_() {} + + Fixed_list_alloc(const Fixed_list_alloc&) = delete; + Fixed_list_alloc& operator=(const Fixed_list_alloc&) = delete; + + storage_type& storage() noexcept + { return store_; } + + template struct rebind { + using other = Fixed_list_alloc<_Up, N, alignment>; + }; + + T* allocate(std::size_t n) + { + return reinterpret_cast(store_.template allocate(n*sizeof(T))); + } + + void deallocate(T* p, std::size_t n) noexcept + { + store_.deallocate(reinterpret_cast(p), n*sizeof(T)); + } + + bool operator==(const Fixed_list_alloc& other) const noexcept + { return this == &other; } + + bool operator!=(const Fixed_list_alloc& other) const noexcept + { return !(*this == other); } + + template + friend bool operator==(const Fixed_list_alloc& x, + const Fixed_list_alloc& y) noexcept; + + template friend class Fixed_list_alloc; + +}; // < class Fixed_list_alloc + +template +inline bool operator==(const Fixed_list_alloc& x, + const Fixed_list_alloc& y) noexcept +{ + return false; // prevent deallocating eachothers resources +} + +template +inline bool operator!=(const Fixed_list_alloc& x, + const Fixed_list_alloc& y) noexcept +{ + return !(x == y); +} + +#endif diff --git a/api/util/fixed_storage.hpp b/api/util/fixed_storage.hpp new file mode 100644 index 0000000000..26889e9e7c --- /dev/null +++ b/api/util/fixed_storage.hpp @@ -0,0 +1,134 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef UTIL_FIXED_STORAGE_HPP +#define UTIL_FIXED_STORAGE_HPP + +#include +#include +#include + +// Implemenation based upon Howard Hinnant's terrific short_alloc example +// https://howardhinnant.github.io/short_alloc.h + +struct Fixed_storage_error : public std::runtime_error { + using runtime_error::runtime_error; +}; + +/** + * @brief Fixed aligned storage (to be used in allocator). + * Pre-allocates enough storage for the number of aligned objects. + * Takes a object type, but underlying storage is just a char array. + * + * @tparam T What type of object to be stored + * @tparam N Number of objects that can be stored + * @tparam alignment Pointer alignment + */ +template +class Fixed_storage { +public: + static constexpr std::size_t aligned_size() noexcept + { return (sizeof(T) + (alignment-1)) & ~(alignment-1); } + + static constexpr std::size_t buffer_size() noexcept + { return N * aligned_size(); } + + struct alignas(alignment) aligned_array : public std::array {}; + using storage_type = aligned_array; + +private: + storage_type buf_; + /** Available addresses */ + Fixed_vector free_; + +public: + Fixed_storage() noexcept; + + template char* allocate(std::size_t n); + void deallocate(char* p, std::size_t n) noexcept; + + static constexpr std::size_t size() noexcept + { return N; } + + std::size_t used() const noexcept + { return buffer_size() - available(); } + + std::size_t available() const noexcept + { return (free_.size() * aligned_size()); } + + void reset() noexcept; + +private: + constexpr const char* end() const noexcept + { return &buf_[buffer_size()]; } + + bool pointer_in_buffer(char* p) const noexcept + { return buf_.data() <= p && p <= end(); } + +}; // < class Fixed_storage + +template +Fixed_storage::Fixed_storage() noexcept +{ + printf("Fixed_storage<%s, %u, %u> aligned_size: %u\n", + typeid(T).name(), N, alignment, aligned_size()); + reset(); +} + +template +template +char* Fixed_storage::allocate(std::size_t n) +{ + static_assert(ReqAlign <= alignment, + "Alignment is too small for this storage"); + + Expects(n == sizeof(T) && + "Allocating more than sizeof(T) not supported"); + + if(UNLIKELY(free_.empty())) + throw Fixed_storage_error{ + "Fixed_storage_error: Storage exhausted (all addresses in use)"}; + + return free_.pop_back(); +} + +template +void Fixed_storage::deallocate(char* p, std::size_t n) noexcept +{ + Expects(pointer_in_buffer(p) && + "Trying to deallocate pointer outside my buffer"); + + free_.push_back(p); + + Ensures(free_.size() <= N); +} + +template +void Fixed_storage::reset() noexcept +{ + auto* ptr = &buf_[N * aligned_size()]; + while(ptr > buf_.data()) + { + ptr -= aligned_size(); + free_.push_back(ptr); + } + + Ensures(free_.size() == N); +} + +#endif diff --git a/test/util/unit/fixed_list_alloc_test.cpp b/test/util/unit/fixed_list_alloc_test.cpp new file mode 100644 index 0000000000..9f2cf09348 --- /dev/null +++ b/test/util/unit/fixed_list_alloc_test.cpp @@ -0,0 +1,61 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +struct Block { + size_t id; + + Block(size_t i) + : id{i} {} +}; + +template +using Block_list = std::list>; + +CASE("Using Fixed_list_alloc") +{ + const int N = 10'000; + Block_list list{{0},{1},{2}}; + + for(int i = 3; i < N; i++) + list.emplace_back(i); + + EXPECT(list.size() == N); + + for(int i = 0; i < N/2; i++) + list.pop_front(); + + EXPECT(list.size() == N/2); + + for(int i = 0; i < N/2; i++) + list.emplace_back(i); + + EXPECT_THROWS_AS(list.emplace_front(322), Fixed_storage_error); + + EXPECT(list.size() == N); + + for(int i = 0; i < N; i++) + list.pop_back(); + + EXPECT(list.empty()); + + for(auto& node : list) + (void) node; +} From 19bdc4b97be5104f6ac06d34bb1de759ae747195 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 18 Oct 2017 10:03:18 +0200 Subject: [PATCH 004/723] test: Build fixed list alloc unit test --- test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 9a51a304d4..eebb147d97 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -103,6 +103,7 @@ set(TEST_SOURCES ${TEST}/util/unit/sha1.cpp ${TEST}/util/unit/crc32.cpp ${TEST}/util/unit/delegate.cpp + ${TEST}/util/unit/fixed_list_alloc_test.cpp ${TEST}/util/unit/fixed_queue.cpp ${TEST}/util/unit/fixed_vector.cpp ${TEST}/util/unit/isotime.cpp From 08914daadeb5ff2c50b43e3d3239282bb98d5dbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 19 Oct 2017 15:04:05 +0200 Subject: [PATCH 005/723] util: Minor cleanup in Fixed_list_alloc --- api/util/fixed_list_alloc.hpp | 9 +++------ api/util/fixed_storage.hpp | 10 ++++------ test/util/unit/fixed_list_alloc_test.cpp | 8 ++++++++ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/api/util/fixed_list_alloc.hpp b/api/util/fixed_list_alloc.hpp index 8b748f8dc4..1bfd4d9ad1 100644 --- a/api/util/fixed_list_alloc.hpp +++ b/api/util/fixed_list_alloc.hpp @@ -55,11 +55,8 @@ class Fixed_list_alloc { Fixed_list_alloc(const Fixed_list_alloc&) = delete; Fixed_list_alloc& operator=(const Fixed_list_alloc&) = delete; - storage_type& storage() noexcept - { return store_; } - - template struct rebind { - using other = Fixed_list_alloc<_Up, N, alignment>; + template struct rebind { + using other = Fixed_list_alloc; }; T* allocate(std::size_t n) @@ -92,7 +89,7 @@ template & x, const Fixed_list_alloc& y) noexcept { - return false; // prevent deallocating eachothers resources + return x == y; } template {}; - using storage_type = aligned_array; private: - storage_type buf_; + alignas(alignment) std::array buf_; /** Available addresses */ - Fixed_vector free_; + Fixed_vector free_; public: Fixed_storage() noexcept; @@ -85,8 +83,8 @@ class Fixed_storage { template Fixed_storage::Fixed_storage() noexcept { - printf("Fixed_storage<%s, %u, %u> aligned_size: %u\n", - typeid(T).name(), N, alignment, aligned_size()); + //printf("Fixed_storage<%s, %u, %u> aligned_size: %u\n", + // typeid(T).name(), N, alignment, aligned_size()); reset(); } diff --git a/test/util/unit/fixed_list_alloc_test.cpp b/test/util/unit/fixed_list_alloc_test.cpp index 9f2cf09348..e538dbf7bf 100644 --- a/test/util/unit/fixed_list_alloc_test.cpp +++ b/test/util/unit/fixed_list_alloc_test.cpp @@ -24,6 +24,9 @@ struct Block { Block(size_t i) : id{i} {} + + bool operator==(const Block& other) const + { return id == other.id; } }; template @@ -51,6 +54,11 @@ CASE("Using Fixed_list_alloc") EXPECT(list.size() == N); + auto it = std::find(list.begin(), list.end(), Block{3}); + list.splice(list.begin(), list, it); + + EXPECT(list.begin() == it); + for(int i = 0; i < N; i++) list.pop_back(); From 783281ba6a0dea3e4872eb09dce1538cb2f6f6c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 19 Oct 2017 15:05:20 +0200 Subject: [PATCH 006/723] tcp: New SACK implementation using std list --- api/net/tcp/sack.hpp | 165 ++++++++++---------------------- test/net/unit/tcp_sack_test.cpp | 12 ++- 2 files changed, 59 insertions(+), 118 deletions(-) diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index 2cfffc3143..b88fb66abd 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -18,6 +18,8 @@ #include #include #include +#include +#include using seq_t = uint32_t; @@ -42,9 +44,6 @@ struct Block { uint64_t whole = 0; }; - Block* older = nullptr; - Block* newer = nullptr; - bool operator==(const Block& other) const noexcept { return start == other.start and end == other.end; } @@ -53,34 +52,15 @@ struct Block { //int32_t precedes (const Block& other) const - bool connect_start(const Block& other) const noexcept + bool connects_start(const Block& other) const noexcept { return other.end == start; } - bool connect_end(const Block& other) const noexcept + bool connects_end(const Block& other) const noexcept { return other.start == end; } Block& operator=(uint64_t whole) { this->whole = whole; return *this; } - - void free() { - detach(); - whole = 0; - older = nullptr; - newer = nullptr; - } - - void detach() { - if (newer) - newer->older = older; - if (older) - older->newer = newer; - - older = nullptr; - newer = nullptr; - } - - }__attribute__((packed)); // Print for Block @@ -116,146 +96,105 @@ class List { }; template -class Array_list { +class Fixed_list { public: + static auto constexpr size = N; + using List = std::list>; + using List_iterator = typename List::iterator; + static_assert(N <= 32 && N > 0, "N wrong sized - optimized for small N"); Ack_result recv_out_of_order(seq_t seq, uint32_t len) { Block inc{seq, seq+len}; - Block* connected_end = nullptr; - Block* connected_start = nullptr; - Block* current = latest_; - - while (current) { + auto connected_end = blocks.end(); + auto connected_start = blocks.end(); - Expects(not current->empty()); + for(auto it = blocks.begin(); it != blocks.end(); it++) + { + Expects(not it->empty()); - if (current->connect_end(inc)) + if (it->connects_end(inc)) { - connected_end = current; + connected_end = it; } - else if (current->connect_start(inc)) + else if (it->connects_start(inc)) { - connected_start = current; + connected_start = it; } - if (connected_start and connected_end) + // if we connected to two nodes, no point in looking for more + if (connected_start != blocks.end() and connected_end != blocks.end()) break; - - current = current->older; - } - Block* update = nullptr; - - // Connectes to two blocks, e.g. fill a hole - if (connected_end) { - - update = connected_end; - update->detach(); - update->end = inc.end; + if (connected_end != blocks.end()) // Connectes to two blocks, e.g. fill a hole + { + connected_end->end = inc.end; // It also connects to a start - if(connected_start) + if (connected_start != blocks.end()) { - update->end = connected_start->end; - connected_start->free(); + connected_end->end = connected_start->end; + blocks.erase(connected_start); } - // Connected only to an end - } else if (connected_start) { - update = connected_start; - update->start = inc.start; - update->detach(); - // No connection - new entry - } else { - update = get_free(); - if (not update) { + move_to_front(connected_end); + } + else if (connected_start != blocks.end()) // Connected only to an start + { + connected_start->start = inc.start; + move_to_front(connected_start); + } + else // No connection - new entry + { + Expects(blocks.size() <= size); + + if(UNLIKELY(blocks.size() == size)) return {recent_entries(), 0}; - } else { - *update = inc; - } + + blocks.push_front(inc); } - update_latest(update); return {recent_entries(), len}; } Ack_result new_valid_ack(seq_t seq) { - Block* current = latest_; uint32_t bytes_freed = 0; - while (current) { - - if (current->start == seq) { - bytes_freed = current->size(); - - if (latest_ == current) - latest_ = current->older; - current->free(); - + for(auto it = blocks.begin(); it != blocks.end(); it++) + { + if (it->start == seq) + { + bytes_freed = it->size(); + blocks.erase(it); break; } - - current = current->older; - }; return {recent_entries(), bytes_freed}; } - void update_latest(Block* blk) + void move_to_front(List_iterator it) { - Expects(blk); - Expects(!blk->empty()); - - if (blk == latest_) - return; - - blk->older = latest_; - blk->newer = nullptr; - - if (latest_) - latest_->newer = blk; - - latest_ = blk; - Ensures(latest_); - Ensures(latest_ != latest_->newer); - Ensures(latest_ != latest_->older); - } - - - Block* get_free() { - - // TODO: Optimize. - for (auto& block : blocks) - if (block.empty()) - return █ - - return nullptr; + if(it != blocks.begin()) + blocks.splice(blocks.begin(), blocks, it); } Entries recent_entries() const { - Entries ret; int i = 0; - Block* current = latest_; - while (current and i < ret.size()) { - ret[i++] = *current; - current = current->older; - } + for(auto it = blocks.begin(); it != blocks.end() and i < ret.size(); it++) + ret[i++] = *it; return ret; } - std::array blocks; - - Block* latest_ = nullptr; + List blocks; }; diff --git a/test/net/unit/tcp_sack_test.cpp b/test/net/unit/tcp_sack_test.cpp index a83c2356da..7579648188 100644 --- a/test/net/unit/tcp_sack_test.cpp +++ b/test/net/unit/tcp_sack_test.cpp @@ -61,7 +61,8 @@ CASE("SACK List Array implementation [RFC 2018]") 8000 5000 5500 8500 8500 5000 5500 9000 */ - List> sack_list; + using Sack_list = List>; + Sack_list sack_list; Ack_result res; // 5000 5500 6000 @@ -114,7 +115,7 @@ CASE("SACK List Array implementation [RFC 2018]") 8000 5500 8000 8500 7000 7500 6000 6500 8500 (lost) */ - sack_list = List>(); + sack_list = Sack_list(); // 5500 6000 6500 res = sack_list.recv_out_of_order(6000, 500); @@ -177,7 +178,8 @@ CASE("SACK block list is full") using namespace net::tcp::sack; constexpr int list_size = 9; - List> sack_list; + using Sack_list = List>; + Sack_list sack_list; Ack_result res; @@ -216,8 +218,8 @@ CASE("SACK block list is full") EXPECT(res.bytes == blksz); // Last block should now be larger - EXPECT(res.entries == expected({seq - incr, seq + blksz}, - {seq - incr * 2, (seq - incr * 2) + blksz }, + EXPECT(res.entries == expected({seq - incr, (seq - incr ) + blksz + blksz }, + {seq - incr * 2, (seq - incr * 2) + blksz }, {seq - incr * 3, (seq - incr * 3) + blksz } )); From 07690fdf748521f75d20b1cc531893920dc8af93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 20 Oct 2017 13:30:13 +0200 Subject: [PATCH 007/723] tcp: More SACK --- api/net/tcp/sack.hpp | 21 ++++++----- test/net/unit/tcp_sack_test.cpp | 62 +++++++++++++++++++++++++-------- 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index b88fb66abd..ec9bbee666 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -50,7 +50,11 @@ struct Block { uint32_t size() const noexcept { return end - start; } bool empty () const noexcept { return whole == 0; } - //int32_t precedes (const Block& other) const + bool contains(const seq_t seq) const noexcept + { + return static_cast(seq - start) >= 0 and + static_cast(seq - end) <= 0; + } bool connects_start(const Block& other) const noexcept { return other.end == start; } @@ -76,23 +80,22 @@ struct Ack_result { uint32_t bytes; }; - template class List { public: + List() = default; + + List(List_impl&& args) + : impl{std::forward(args)} + {} Ack_result recv_out_of_order(seq_t seq, size_t len) - { - return impl.recv_out_of_order(seq, len); - } + { return impl.recv_out_of_order(seq, len); } Ack_result new_valid_ack(seq_t end) - { - return impl.new_valid_ack(end); - } + { return impl.new_valid_ack(end); } List_impl impl; - }; template diff --git a/test/net/unit/tcp_sack_test.cpp b/test/net/unit/tcp_sack_test.cpp index 7579648188..55f06b8227 100644 --- a/test/net/unit/tcp_sack_test.cpp +++ b/test/net/unit/tcp_sack_test.cpp @@ -34,7 +34,32 @@ net::tcp::sack::Entries expected( return {{first, second, third}}; } -CASE("SACK List Array implementation [RFC 2018]") +CASE("Block test") +{ + using namespace net::tcp::sack; + seq_t start {0}; + seq_t end {1500}; + Block block{start, end}; + + EXPECT(block.size() == 1500); + EXPECT(not block.empty()); + EXPECT(block.contains(1000)); + EXPECT(not block.contains(2000)); + + block.start -= 1000; + EXPECT(block.size() == 2500); + EXPECT(block.contains(1000)); + EXPECT(block.contains(-500)); + + block.end -= 1500; + EXPECT(block.size() == 1000); + EXPECT(not block.contains(1000)); + EXPECT(block.contains(0)); + EXPECT(block.contains(-500)); + EXPECT(block.contains(-1000)); +} + +CASE("SACK Fixed List implementation [RFC 2018]") { using namespace std; using namespace net::tcp::sack; @@ -93,7 +118,6 @@ CASE("SACK List Array implementation [RFC 2018]") res = sack_list.recv_out_of_order(8500,500); EXPECT(res.entries == expected({5500, 9000})); - /* Case 3: The 2nd, 4th, 6th, and 8th (last) segments are dropped. @@ -144,14 +168,9 @@ CASE("SACK List Array implementation [RFC 2018]") 6500 5500 6000 7500 8000 8500 */ - for(auto& b : sack_list.impl.blocks) - std::cout << b << "\n"; // 6000 7500 8000 8500 res = sack_list.recv_out_of_order(6500, 500); - for(auto& b : sack_list.impl.blocks) - std::cout << b << "\n"; - EXPECT(res.entries == expected({6000,7500}, {8000,8500})); /* @@ -164,12 +183,27 @@ CASE("SACK List Array implementation [RFC 2018]") 5500 7500 8000 8500 */ - auto result = sack_list.new_valid_ack(5500 + 500); - for(auto& b : sack_list.impl.blocks) - std::cout << b << "\n"; - EXPECT(result.entries == expected({8000,8500})); - EXPECT(result.bytes == 7500-6000); + res = sack_list.new_valid_ack(5500 + 500); + + EXPECT(res.entries == expected({8000,8500})); + EXPECT(res.bytes == 7500-6000); + + + // Create a hole which connects both ends and fill the hole + sack_list = Sack_list(); + res = sack_list.recv_out_of_order(5500, 500); + EXPECT(res.entries == expected({5500,6000})); + + res = sack_list.recv_out_of_order(6500, 500); + EXPECT(res.entries == expected({6500,7000}, {5500,6000})); + + res = sack_list.recv_out_of_order(6000, 500); + EXPECT(res.entries == expected({5500,7000})); + + res = sack_list.new_valid_ack(5500); + EXPECT(res.entries == expected()); + EXPECT(res.bytes == 1500); } CASE("SACK block list is full") @@ -198,9 +232,6 @@ CASE("SACK block list is full") {seq - incr * 2, (seq - incr * 2) + blksz }, {seq - incr * 3, (seq - incr * 3) + blksz } )); - std::cout << "Fully populated SACK list: \n" - << res.entries << "\n"; - EXPECT(res.bytes == blksz); // Try adding one more @@ -211,6 +242,7 @@ CASE("SACK block list is full") {seq - incr * 2, (seq - incr * 2) + blksz }, {seq - incr * 3, (seq - incr * 3) + blksz } )); + // Nothing inserted EXPECT(res.bytes == 0); // Add a block that connects to the end, which should free up one spot From 48470dba231f674b9555f627fa43d536e8415096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 24 Oct 2017 10:37:58 +0200 Subject: [PATCH 008/723] tcp: Added SACK scoreboard + tests --- api/net/tcp/sack.hpp | 180 +++++++++++++++++++++++++------- test/net/unit/tcp_sack_test.cpp | 127 ++++++++++++++++++++++ 2 files changed, 267 insertions(+), 40 deletions(-) diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index ec9bbee666..8e0cb063ad 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -56,6 +56,12 @@ struct Block { static_cast(seq - end) <= 0; } + bool precedes(const seq_t seq) const noexcept + { return static_cast(end - seq) <= 0; } + + bool precedes(const seq_t seq, const Block& other) const noexcept + { return (start - seq) < (other.start - seq); } + bool connects_start(const Block& other) const noexcept { return other.end == start; } @@ -89,15 +95,46 @@ class List { : impl{std::forward(args)} {} - Ack_result recv_out_of_order(seq_t seq, size_t len) - { return impl.recv_out_of_order(seq, len); } + Ack_result recv_out_of_order(const seq_t seq, const size_t len) + { return impl.recv_out_of_order({seq, static_cast(seq+len)}); } - Ack_result new_valid_ack(seq_t end) + Ack_result new_valid_ack(const seq_t end) { return impl.new_valid_ack(end); } List_impl impl; }; +template +struct Connect_result { + Iterator end; + Iterator start; +}; + +template +Connect_result +connects_to(Iterator first, Iterator last, const Connectable& value) +{ + Connect_result connected{last, last}; + for (; first != last; ++first) + { + Expects(not first->empty()); + + if (first->connects_end(value)) + { + connected.end = first; + } + else if (first->connects_start(value)) + { + connected.start = first; + } + + // if we connected to two nodes, no point in looking for more + if (connected.start != last and connected.end != last) + break; + } + return connected; +} + template class Fixed_list { public: @@ -107,48 +144,27 @@ class Fixed_list { static_assert(N <= 32 && N > 0, "N wrong sized - optimized for small N"); - Ack_result recv_out_of_order(seq_t seq, uint32_t len) + Ack_result recv_out_of_order(Block blk) { - Block inc{seq, seq+len}; - - auto connected_end = blocks.end(); - auto connected_start = blocks.end(); - - for(auto it = blocks.begin(); it != blocks.end(); it++) - { - Expects(not it->empty()); + auto connected = connects_to(blocks.begin(), blocks.end(), blk); - if (it->connects_end(inc)) - { - connected_end = it; - } - else if (it->connects_start(inc)) - { - connected_start = it; - } - - // if we connected to two nodes, no point in looking for more - if (connected_start != blocks.end() and connected_end != blocks.end()) - break; - } - - if (connected_end != blocks.end()) // Connectes to two blocks, e.g. fill a hole + if (connected.end != blocks.end()) // Connectes to an end { - connected_end->end = inc.end; + connected.end->end = blk.end; - // It also connects to a start - if (connected_start != blocks.end()) + // It also connects to a start, e.g. fills a hole + if (connected.start != blocks.end()) { - connected_end->end = connected_start->end; - blocks.erase(connected_start); + connected.end->end = connected.start->end; + blocks.erase(connected.start); } - move_to_front(connected_end); + move_to_front(connected.end); } - else if (connected_start != blocks.end()) // Connected only to an start + else if (connected.start != blocks.end()) // Connected only to an start { - connected_start->start = inc.start; - move_to_front(connected_start); + connected.start->start = blk.start; + move_to_front(connected.start); } else // No connection - new entry { @@ -157,13 +173,13 @@ class Fixed_list { if(UNLIKELY(blocks.size() == size)) return {recent_entries(), 0}; - blocks.push_front(inc); + blocks.push_front(blk); } - return {recent_entries(), len}; + return {recent_entries(), blk.size()}; } - Ack_result new_valid_ack(seq_t seq) + Ack_result new_valid_ack(const seq_t seq) { uint32_t bytes_freed = 0; @@ -175,7 +191,7 @@ class Fixed_list { blocks.erase(it); break; } - }; + } return {recent_entries(), bytes_freed}; } @@ -201,6 +217,90 @@ class Fixed_list { }; +// SCOREBOARD stuff + +template +class Scoreboard { +public: + void recv_sack(const seq_t current, Block blk) + { return impl.recv_sack(current, blk); } + + void recv_sack(const seq_t current, seq_t start, seq_t end) + { return recv_sack(current, {start, end}); } + + void new_valid_ack(const seq_t seq) + { return impl.new_valid_ack(seq); } + + void clear() + { impl.clear(); } + + Scoreboard_impl impl; +}; + +template +class Scoreboard_list { +public: + static auto constexpr size = N; + using List = std::list>; + using List_iterator = typename List::iterator; + + void recv_sack(const seq_t current, Block blk) + { + auto connected = connects_to(blocks.begin(), blocks.end(), blk); + + if (connected.end != blocks.end()) // Connectes to an end + { + connected.end->end = blk.end; + + // It also connects to a start, e.g. fills a hole + if (connected.start != blocks.end()) + { + connected.end->end = connected.start->end; + blocks.erase(connected.start); + } + } + else if (connected.start != blocks.end()) // Connected only to an start + { + connected.start->start = blk.start; + } + else // No connection - new entry + { + insert(current, blk); + } + } + + void new_valid_ack(const seq_t seq) + { + for(auto it = blocks.begin(); it != blocks.end(); ) + { + auto tmp = it++; + + if (tmp->precedes(seq)) + blocks.erase(tmp); + } + } + + void clear() + { blocks.clear(); } + + void insert(const seq_t current, Block blk) + { + Expects(blocks.size() <= size); + + if(UNLIKELY(blocks.size() == size)) + return; + + auto it = std::find_if(blocks.begin(), blocks.end(), + [&](const auto& block) { + return blk.precedes(current, block); + }); + + blocks.insert(it, blk); + } + + List blocks; +}; + } // < namespace sack } // < namespace tcp } // < namespace net diff --git a/test/net/unit/tcp_sack_test.cpp b/test/net/unit/tcp_sack_test.cpp index 55f06b8227..8bd1a81d7f 100644 --- a/test/net/unit/tcp_sack_test.cpp +++ b/test/net/unit/tcp_sack_test.cpp @@ -34,6 +34,7 @@ net::tcp::sack::Entries expected( return {{first, second, third}}; } +#include CASE("Block test") { using namespace net::tcp::sack; @@ -57,6 +58,39 @@ CASE("Block test") EXPECT(block.contains(0)); EXPECT(block.contains(-500)); EXPECT(block.contains(-1000)); + + const uint32_t max_uint = std::numeric_limits::max(); + block.start = max_uint - 10; + block.end = 10; + EXPECT(block.contains(max_uint)); + EXPECT(block.contains(5)); + + block.start = max_uint; + block.end = max_uint / 2 + 1; + EXPECT(block.size() > max_uint / 2); + // < breaks apart when size() > max_uint/2 + EXPECT(not block.contains(max_uint / 2)); + + block.start = max_uint; + block.end = max_uint / 2 - 1; + EXPECT(block.size() == max_uint / 2); + EXPECT(block.contains(max_uint / 2 - 1)); + + + const seq_t seq = 1000; + Block blk_lesser{1500,2000}; + Block blk_greater{2500,3000}; + Block blk_greater2{0,1000}; + + EXPECT(blk_lesser.precedes(seq, blk_greater)); + EXPECT(blk_lesser.precedes(seq, blk_greater2)); + EXPECT(blk_greater.precedes(seq, blk_greater2)); + + EXPECT(not blk_lesser.precedes(seq)); + EXPECT(not blk_lesser.precedes(1500)); + EXPECT(not blk_lesser.precedes(1800)); + EXPECT(blk_lesser.precedes(2000)); + EXPECT(blk_lesser.precedes(4000)); } CASE("SACK Fixed List implementation [RFC 2018]") @@ -258,3 +292,96 @@ CASE("SACK block list is full") // Add a block that connects two blocks, which should free up one spot } + +CASE("SACK Scoreboard - recv SACK") +{ + using namespace net::tcp::sack; + using Sack_scoreboard = Scoreboard>; + + Sack_scoreboard scoreboard; + const seq_t current = 1000; + auto& blocks = scoreboard.impl.blocks; + auto it = blocks.end(); + EXPECT(blocks.empty()); + + scoreboard.recv_sack(current, 1500, 2000); + EXPECT(blocks.size() == 1); + + scoreboard.recv_sack(current, 3500, 4000); + EXPECT(blocks.size() == 2); + + it = blocks.begin(); + EXPECT(*it == Block(1500,2000)); + it++; + EXPECT(*it == Block(3500,4000)); + + scoreboard.recv_sack(current, 2500, 3000); + EXPECT(blocks.size() == 3); + + it = blocks.begin(); + EXPECT(*it == Block(1500,2000)); + it++; + EXPECT(*it == Block(2500, 3000)); + it++; + EXPECT(*it == Block(3500, 4000)); + + scoreboard.recv_sack(current, 2000, 2500); + EXPECT(blocks.size() == 2); + + it = blocks.begin(); + EXPECT(*it == Block(1500,3000)); + it++; + EXPECT(*it == Block(3500,4000)); + + scoreboard.recv_sack(current, 3000, 3500); + EXPECT(blocks.size() == 1); + + it = blocks.begin(); + EXPECT(*it == Block(1500,4000)); + + scoreboard.clear(); + + EXPECT(blocks.empty()); +} + +CASE("SACK Scoreboard - new valid ACK") +{ + using namespace net::tcp::sack; + using Sack_scoreboard = Scoreboard>; + + Sack_scoreboard scoreboard; + const seq_t current = 1000; + auto& blocks = scoreboard.impl.blocks; + auto it = blocks.end(); + EXPECT(blocks.empty()); + + scoreboard.recv_sack(current, 1500, 2000); + scoreboard.recv_sack(current, 2500, 3000); + scoreboard.recv_sack(current, 3500, 4000); + + scoreboard.new_valid_ack(1000); + EXPECT(blocks.size() == 3); + + it = blocks.begin(); + EXPECT(*it == Block(1500,2000)); + it++; + EXPECT(*it == Block(2500,3000)); + it++; + EXPECT(*it == Block(3500,4000)); + + scoreboard.new_valid_ack(2000); + EXPECT(blocks.size() == 2); + + it = blocks.begin(); + EXPECT(*it == Block(2500,3000)); + it++; + EXPECT(*it == Block(3500,4000)); + + scoreboard.new_valid_ack(4000); + EXPECT(blocks.empty()); +} + +CASE("SACK Scoreboard - partial ACK") +{ + // Currently not supported (don't know if needed, yet) +} From d8835c54cc8cd77b8461d03ce5c2953c77da5db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 24 Oct 2017 13:09:57 +0200 Subject: [PATCH 009/723] util: Added more functionality to Fixed_vector --- api/util/fixed_vector.hpp | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/api/util/fixed_vector.hpp b/api/util/fixed_vector.hpp index d6a3d1ec65..45ced8e3c3 100644 --- a/api/util/fixed_vector.hpp +++ b/api/util/fixed_vector.hpp @@ -34,9 +34,17 @@ enum class Fixedvector_Init { template struct Fixed_vector { + using value_type = T; Fixed_vector() : count(0) {} Fixed_vector(Fixedvector_Init) {} + Fixed_vector(std::initializer_list l) + : count(l.size()) + { + Expects(count <= capacity()); + std::memcpy(begin(), l.begin(), count * sizeof(T)); + } + // add existing T& push_back(const T& e) noexcept { assert(count < N); @@ -75,6 +83,9 @@ struct Fixed_vector { return (T*) (element + i); } + T* data() noexcept { + return (T*) &element[0]; + } T* begin() noexcept { return (T*) &element[0]; } @@ -82,6 +93,16 @@ struct Fixed_vector { return (T*) &element[count]; } + const T* data() const noexcept { + return (T*) &element[0]; + } + const T* begin() const noexcept { + return (T*) &element[0]; + } + const T* end() const noexcept { + return (T*) &element[count]; + } + T& back() noexcept { assert(not empty()); return (T&)element[count-1]; @@ -102,6 +123,12 @@ struct Fixed_vector { count = size; } + template + bool operator==(const R& rhs) const noexcept + { + return size() == rhs.size() and std::memcmp(data(), rhs.data(), size()*sizeof(T)) == 0; + } + private: uint32_t count; typename std::aligned_storage::type element[N]; From c339276dec712e8825550d67b2126e039870cdde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 24 Oct 2017 13:10:57 +0200 Subject: [PATCH 010/723] tcp: SACK result is now Fixed_vector instead of array --- api/net/tcp/sack.hpp | 14 +++++--- test/net/unit/tcp_sack_test.cpp | 57 +++++++++++++++++---------------- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index 8e0cb063ad..9a0fb2adb6 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -44,6 +44,12 @@ struct Block { uint64_t whole = 0; }; + void swap_endian() noexcept + { + start = htonl(start); + end = htonl(end); + } + bool operator==(const Block& other) const noexcept { return start == other.start and end == other.end; } @@ -79,7 +85,7 @@ std::ostream& operator<<(std::ostream& out, const Block& b) { return out; } -using Entries = std::array; +using Entries = Fixed_vector; struct Ack_result { Entries entries; @@ -117,8 +123,6 @@ connects_to(Iterator first, Iterator last, const Connectable& value) Connect_result connected{last, last}; for (; first != last; ++first) { - Expects(not first->empty()); - if (first->connects_end(value)) { connected.end = first; @@ -207,8 +211,8 @@ class Fixed_list { Entries ret; int i = 0; - for(auto it = blocks.begin(); it != blocks.end() and i < ret.size(); it++) - ret[i++] = *it; + for(auto it = blocks.begin(); it != blocks.end() and ret.size() < ret.capacity(); it++) + ret.push_back(*it); return ret; } diff --git a/test/net/unit/tcp_sack_test.cpp b/test/net/unit/tcp_sack_test.cpp index 8bd1a81d7f..857790334c 100644 --- a/test/net/unit/tcp_sack_test.cpp +++ b/test/net/unit/tcp_sack_test.cpp @@ -26,12 +26,9 @@ std::ostream& operator<< (std::ostream& out, const net::tcp::sack::Entries& ent) return out; } -net::tcp::sack::Entries expected( - net::tcp::sack::Block first = net::tcp::sack::Block(), - net::tcp::sack::Block second = net::tcp::sack::Block(), - net::tcp::sack::Block third = net::tcp::sack::Block()) +net::tcp::sack::Entries expected(std::initializer_list blocks) { - return {{first, second, third}}; + return {blocks}; } #include @@ -91,6 +88,12 @@ CASE("Block test") EXPECT(not blk_lesser.precedes(1800)); EXPECT(blk_lesser.precedes(2000)); EXPECT(blk_lesser.precedes(4000)); + + // NOTE: Invalidates the block + block = Block{1500, 2000}; + block.swap_endian(); + EXPECT(block.start == htonl(1500)); + EXPECT(block.end == htonl(2000)); } CASE("SACK Fixed List implementation [RFC 2018]") @@ -126,31 +129,31 @@ CASE("SACK Fixed List implementation [RFC 2018]") // 5000 5500 6000 res = sack_list.recv_out_of_order(5500, 500); - EXPECT(res.entries == expected({5500,6000})); + EXPECT(res.entries == expected({{5500,6000}})); // 5000 5500 6500 res = sack_list.recv_out_of_order(6000, 500); - EXPECT(res.entries == expected({5500,6500})); + EXPECT(res.entries == expected({{5500,6500}})); // 5000 5500 7000 res = sack_list.recv_out_of_order(6500, 500); - EXPECT(res.entries == expected({5500,7000})); + EXPECT(res.entries == expected({{5500,7000}})); // 5000 5500 7500 res = sack_list.recv_out_of_order(7000,500); - EXPECT(res.entries == expected({5500,7500})); + EXPECT(res.entries == expected({{5500,7500}})); // 5000 5500 8000 res = sack_list.recv_out_of_order(7500,500); - EXPECT(res.entries == expected({5500,8000})); + EXPECT(res.entries == expected({{5500,8000}})); // 5000 5500 8500 res = sack_list.recv_out_of_order(8000,500); - EXPECT(res.entries == expected({5500,8500})); + EXPECT(res.entries == expected({{5500,8500}})); // 5000 5500 9000 res = sack_list.recv_out_of_order(8500,500); - EXPECT(res.entries == expected({5500, 9000})); + EXPECT(res.entries == expected({{5500, 9000}})); /* Case 3: The 2nd, 4th, 6th, and 8th (last) segments are @@ -177,15 +180,15 @@ CASE("SACK Fixed List implementation [RFC 2018]") // 5500 6000 6500 res = sack_list.recv_out_of_order(6000, 500); - EXPECT(res.entries == expected({6000,6500})); + EXPECT(res.entries == expected({{6000,6500}})); // 5500 7000 7500 6000 6500 res = sack_list.recv_out_of_order(7000, 500); - EXPECT(res.entries == expected({7000,7500}, {6000,6500})); + EXPECT(res.entries == expected({{7000,7500}, {6000,6500}})); // 5500 8000 8500 7000 7500 6000 6500 res = sack_list.recv_out_of_order(8000, 500); - EXPECT(res.entries == expected({8000,8500}, {7000,7500}, {6000,6500})); + EXPECT(res.entries == expected({{8000,8500}, {7000,7500}, {6000,6500}})); /* Suppose at this point, the 4th packet is received out of order. @@ -205,7 +208,7 @@ CASE("SACK Fixed List implementation [RFC 2018]") // 6000 7500 8000 8500 res = sack_list.recv_out_of_order(6500, 500); - EXPECT(res.entries == expected({6000,7500}, {8000,8500})); + EXPECT(res.entries == expected({{6000,7500}, {8000,8500}})); /* Suppose at this point, the 2nd segment is received. The data @@ -219,7 +222,7 @@ CASE("SACK Fixed List implementation [RFC 2018]") */ res = sack_list.new_valid_ack(5500 + 500); - EXPECT(res.entries == expected({8000,8500})); + EXPECT(res.entries == expected({{8000,8500}})); EXPECT(res.bytes == 7500-6000); @@ -227,16 +230,16 @@ CASE("SACK Fixed List implementation [RFC 2018]") sack_list = Sack_list(); res = sack_list.recv_out_of_order(5500, 500); - EXPECT(res.entries == expected({5500,6000})); + EXPECT(res.entries == expected({{5500,6000}})); res = sack_list.recv_out_of_order(6500, 500); - EXPECT(res.entries == expected({6500,7000}, {5500,6000})); + EXPECT(res.entries == expected({{6500,7000}, {5500,6000}})); res = sack_list.recv_out_of_order(6000, 500); - EXPECT(res.entries == expected({5500,7000})); + EXPECT(res.entries == expected({{5500,7000}})); res = sack_list.new_valid_ack(5500); - EXPECT(res.entries == expected()); + EXPECT(res.entries == expected({})); EXPECT(res.bytes == 1500); } @@ -262,9 +265,9 @@ CASE("SACK block list is full") } // Fully populated sack list - EXPECT(res.entries == expected({seq - incr, (seq - incr ) + blksz}, + EXPECT(res.entries == expected({{seq - incr, (seq - incr ) + blksz}, {seq - incr * 2, (seq - incr * 2) + blksz }, - {seq - incr * 3, (seq - incr * 3) + blksz } )); + {seq - incr * 3, (seq - incr * 3) + blksz } })); EXPECT(res.bytes == blksz); @@ -272,9 +275,9 @@ CASE("SACK block list is full") res = sack_list.recv_out_of_order(seq, blksz); // We should now get the same - EXPECT(res.entries == expected({seq - incr, (seq - incr ) + blksz}, + EXPECT(res.entries == expected({{seq - incr, (seq - incr ) + blksz}, {seq - incr * 2, (seq - incr * 2) + blksz }, - {seq - incr * 3, (seq - incr * 3) + blksz } )); + {seq - incr * 3, (seq - incr * 3) + blksz } })); // Nothing inserted EXPECT(res.bytes == 0); @@ -284,9 +287,9 @@ CASE("SACK block list is full") EXPECT(res.bytes == blksz); // Last block should now be larger - EXPECT(res.entries == expected({seq - incr, (seq - incr ) + blksz + blksz }, + EXPECT(res.entries == expected({{seq - incr, (seq - incr ) + blksz + blksz }, {seq - incr * 2, (seq - incr * 2) + blksz }, - {seq - incr * 3, (seq - incr * 3) + blksz } )); + {seq - incr * 3, (seq - incr * 3) + blksz } })); // Add a block that connects two blocks, which should free up one spot From 5dead40adf7cd217698507b67f315d58bcada773 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 24 Oct 2017 13:48:53 +0200 Subject: [PATCH 011/723] tcp: Added SACK options --- api/net/tcp/options.hpp | 82 +++++++++++++++++++++++++++++------------ api/net/tcp/packet.hpp | 4 +- 2 files changed, 61 insertions(+), 25 deletions(-) diff --git a/api/net/tcp/options.hpp b/api/net/tcp/options.hpp index 37a038cf1c..a6e6203202 100644 --- a/api/net/tcp/options.hpp +++ b/api/net/tcp/options.hpp @@ -30,11 +30,13 @@ namespace tcp { struct Option { enum Kind { - END = 0x00, // End of option list - NOP = 0x01, // No-Operation - MSS = 0x02, // Maximum Segment Size [RFC 793] Rev: [879, 6691] - WS = 0x03, // Window Scaling [RFC 7323] p. 8 - TS = 0x08, // Timestamp [RFC 7323] p. 11 + END = 0x00, // End of option list + NOP = 0x01, // No-Operation + MSS = 0x02, // Maximum Segment Size [RFC 793] Rev: [879, 6691] + WS = 0x03, // Window Scaling [RFC 7323] p. 8 + SACK_PERM = 0x04, // Sack-Permitted [RFC 2018] + SACK = 0x05, // Selective ACK [RFC 2018] + TS = 0x08, // Timestamp [RFC 7323] p. 11 }; const uint8_t kind {END}; @@ -45,24 +47,16 @@ struct Option { { return kind_string(static_cast(kind)); } static std::string kind_string(Kind kind) { - switch(kind) { - case MSS: - return {"MSS"}; - - case WS: - return {"Window Scaling"}; - - case TS: - return {"Timestamp"}; - - case NOP: - return {"No-Operation"}; - - case END: - return {"End of list"}; - - default: - return {"Unknown Option"}; + switch(kind) + { + case MSS: return {"MSS"}; + case WS: return {"Window Scaling"}; + case SACK_PERM: return {"SACK Permitted"}; + case SACK: return {"SACK"}; + case TS: return {"Timestamp"}; + case NOP: return {"No-Operation"}; + case END: return {"End of list"}; + default: return {"Unknown Option"}; } } @@ -140,6 +134,48 @@ struct Option { } __attribute__((packed)); + /** + * @brief SACK Permitted [RFC 2018] p. 11 + */ + struct opt_sack_perm { + const uint8_t kind {SACK_PERM}; + const uint8_t length {2}; + + opt_sack_perm() = default; + + } __attribute__((packed)); + + /** + * @brief Timestamp option [RFC 7323] p. 11 + */ + struct opt_sack { + const uint8_t kind{SACK}; + const uint8_t length; + uint8_t val[0]; + + template + opt_sack(const Collection& blocks) + : length{static_cast(2 + + (blocks.size() * sizeof(typename Collection::value_type)))} + { + using T = typename Collection::value_type; + Expects(blocks.size() * sizeof(T) <= 4*8); + std::memcpy(&val[0], blocks.data(), blocks.size() * sizeof(T)); + } + + } __attribute__((packed)); + + struct opt_sack_align { + const uint8_t padding[2] {NOP, NOP}; + const opt_sack sack; + + template + opt_sack_align(const Collection& blocks) + : sack{blocks} + {} + + } __attribute__((packed)); + }; // < struct Option } // < namespace net diff --git a/api/net/tcp/packet.hpp b/api/net/tcp/packet.hpp index bd7190275f..6ae1f261be 100644 --- a/api/net/tcp/packet.hpp +++ b/api/net/tcp/packet.hpp @@ -84,7 +84,7 @@ class Packet : public PacketIP4 { uint16_t tcp_checksum() const noexcept { return tcp_header().checksum; } -Socket source() const + Socket source() const { return Socket{ip_src(), src_port()}; } Socket destination() const @@ -240,7 +240,7 @@ Socket source() const /** * @brief Adds a tcp option aligned. - * Assumes the user knows what he's doing. + * Assumes the user knows what she/he is doing. * * @tparam T An aligned TCP option * @tparam Args construction args to option T From b464277f00c4682d669cf64ca68bad275b1f636c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 28 Nov 2017 13:36:58 +0100 Subject: [PATCH 012/723] tcp: Implemented SACK handshake (SACK permitted) --- api/net/tcp/common.hpp | 2 ++ api/net/tcp/connection.hpp | 5 +++++ api/net/tcp/tcp.hpp | 18 ++++++++++++++++++ src/net/tcp/connection.cpp | 29 +++++++++++++++++++++++++++++ src/net/tcp/connection_states.cpp | 11 +++++++++++ src/net/tcp/tcp.cpp | 1 + 6 files changed, 66 insertions(+) diff --git a/api/net/tcp/common.hpp b/api/net/tcp/common.hpp index 18478bc6ee..80c30d3167 100644 --- a/api/net/tcp/common.hpp +++ b/api/net/tcp/common.hpp @@ -35,6 +35,8 @@ namespace net { static constexpr uint32_t default_ws_window_size {8192 << default_window_scaling}; // use of timestamps option static constexpr bool default_timestamps {true}; + // use of SACK + static constexpr bool default_sack {true}; // maximum size of a TCP segment - later set based on MTU or peer static constexpr uint16_t default_mss {536}; // the maximum amount of half-open connections per port (listener) diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index 47511bca66..569bdafb01 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -846,6 +846,9 @@ class Connection { /** State if connection is in TCP write queue or not. */ bool queued_; + /** If SACK is permitted (option has been seen from peer) */ + bool sack_perm = false; + /** Congestion control */ // is fast recovery state bool fast_recovery_ = false; @@ -1021,6 +1024,8 @@ class Connection { bool uses_timestamps() const noexcept; + bool uses_SACK() const noexcept; + /// --- INCOMING / TRANSMISSION --- /// /* Receive a TCP Packet. diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index facb1d1f3c..3261bdd697 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -305,6 +305,22 @@ namespace net { bool uses_timestamps() const { return timestamps_; } + /** + * @brief Sets if SACK Option is gonna be used. + * + * @param[in] active Whether SACK Option are in use. + */ + void set_SACK(bool active) noexcept + { sack_ = active; } + + /** + * @brief Whether the TCP instance is using SACK Options or not. + * + * @return Whether the TCP instance is using SACK Options or not. + */ + bool uses_SACK() const noexcept + { return sack_; } + /** * @brief Sets the dack. [RFC 1122] (p.96) * @@ -478,6 +494,8 @@ namespace net { uint8_t wscale_; /** Timestamp option active [RFC 7323] p. 11 */ bool timestamps_; + /** Selective ACK [RFC 2018] */ + bool sack_; /** Delayed ACK timeout - how long should we wait with sending an ACK */ std::chrono::milliseconds dack_timeout_; /** Maximum SYN queue backlog */ diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 4f3762df9d..285c58de5e 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -985,6 +985,25 @@ void Connection::parse_options(const Packet& packet) { break; } + case Option::SACK_PERM: + { + printf("\n"); + + if(UNLIKELY(option->length != sizeof(Option::opt_sack_perm))) + throw TCPBadOptionException{Option::SACK_PERM, "length != 2"}; + + if(UNLIKELY(!packet.isset(SYN))) + throw TCPBadOptionException{Option::SACK_PERM, "Non-SYN packet"}; + + if(host_.uses_SACK()) + { + sack_perm = true; + } + + opt += option->length; + break; + } + default: opt += option->length; break; @@ -1013,6 +1032,11 @@ void Connection::add_option(Option::Kind kind, Packet& packet) { packet.add_tcp_option(host_.get_ts_value(), ts_ecr); break; } + + case Option::SACK_PERM: { + packet.add_tcp_option(); + break; + } default: break; } @@ -1038,6 +1062,11 @@ bool Connection::uses_timestamps() const noexcept return host_.uses_timestamps(); } +bool Connection::uses_SACK() const noexcept +{ + return host_.uses_SACK(); +} + void Connection::drop(const Packet& packet, Drop_reason reason) { signal_packet_dropped(packet, reason); diff --git a/src/net/tcp/connection_states.cpp b/src/net/tcp/connection_states.cpp index 017d5d554a..f067116692 100644 --- a/src/net/tcp/connection_states.cpp +++ b/src/net/tcp/connection_states.cpp @@ -541,6 +541,11 @@ void Connection::Closed::open(Connection& tcp, bool active) { tcp.add_option(Option::TS, *packet); } + if(tcp.uses_SACK()) + { + tcp.add_option(Option::SACK_PERM, *packet); + } + tcb.SND.UNA = tcb.ISS; tcb.SND.NXT = tcb.ISS+1; tcp.transmit(std::move(packet)); @@ -828,6 +833,12 @@ State::Result Connection::Listen::handle(Connection& tcp, Packet_ptr in) { packet->set_win(std::min((uint32_t)default_window_size, tcb.RCV.WND)); } + // SACK permitted + if(tcp.sack_perm == true) + { + tcp.add_option(Option::SACK_PERM, *packet); + } + tcp.transmit(std::move(packet)); tcp.set_state(SynReceived::instance()); diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index f6aa660929..e2f63f242c 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -38,6 +38,7 @@ TCP::TCP(IPStack& inet, bool smp_enable) : win_size_{default_ws_window_size}, // 8096*1024 wscale_{default_window_scaling}, // 5 timestamps_{default_timestamps}, // true + sack_{default_sack}, // true dack_timeout_{default_dack_timeout}, // 40ms max_syn_backlog_{default_max_syn_backlog} // 64 { From 03b02474f110f6e6fe63e0adc1ee14b067525f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 30 Nov 2017 11:19:17 +0100 Subject: [PATCH 013/723] tcp: Moved process segment logic closer to connection & simplified sequence check --- api/net/tcp/connection.hpp | 9 ++- src/net/tcp/connection.cpp | 79 ++++++++++++++++++ src/net/tcp/connection_states.cpp | 129 ++++-------------------------- 3 files changed, 101 insertions(+), 116 deletions(-) diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index 569bdafb01..bbb1f5abd0 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -671,8 +671,6 @@ class Connection { virtual bool check_ack(Connection&, const Packet&); - virtual void process_segment(Connection&, Packet&); - virtual void process_fin(Connection&, const Packet&); virtual void send_reset(Connection&); @@ -1038,6 +1036,13 @@ class Connection { */ bool handle_ack(const Packet&); + /** + * @brief Receive data from an incoming packet containing data. + * + * @param[in] in TCP Packet containing payload + */ + void recv_data(const Packet& in); + /** * @brief Determines if the incoming segment is a legit window update. * diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 285c58de5e..5dfd01f1df 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -629,6 +629,85 @@ void Connection::rtx_ack(const seq_t ack) { // x-rtx_q.size(), rtx_q.size()); } +/* + 7. Process the segment text + + If a packet has data, process the data. + + [RFC 793] Page 74: + + Once in the ESTABLISHED state, it is possible to deliver segment + text to user RECEIVE buffers. Text from segments can be moved + into buffers until either the buffer is full or the segment is + empty. If the segment empties and carries an PUSH flag, then + the user is informed, when the buffer is returned, that a PUSH + has been received. + + When the TCP takes responsibility for delivering the data to the + user it must also acknowledge the receipt of the data. + + Once the TCP takes responsibility for the data it advances + RCV.NXT over the data accepted, and adjusts RCV.WND as + apporopriate to the current buffer availability. The total of + RCV.NXT and RCV.WND should not be reduced. + + Please note the window management suggestions in section 3.7. + + Send an acknowledgment of the form: + + + + This acknowledgment should be piggybacked on a segment being + transmitted if possible without incurring undue delay. +*/ +void Connection::recv_data(const Packet& in) +{ + Expects(in.has_tcp_data()); + const auto length = in.tcp_data_length(); + cb.RCV.NXT += length; + const auto snd_nxt = cb.SND.NXT; + if(read_request and read_request->buffer.capacity()) { + auto received = receive(in.seq(), in.tcp_data(), in.tcp_data_length(), in.isset(PSH)); + Ensures(received == length); + } + + // [RFC 5681] + //tcb.SND.cwnd += std::min(length, tcp.SMSS()); + + // user callback didnt result in transmitting an ACK + if (cb.SND.NXT == snd_nxt) + { + // ACK by trying to send more + if (can_send()) + { + writeq_push(); + // nothing got sent + if (cb.SND.NXT == snd_nxt) + { + send_ack(); + } + // something got sent + else + { + dack_ = 0; + } + } + // else regular ACK + else + { + if (use_dack() and dack_ == 0) + { + start_dack(); + } + else + { + stop_dack(); + send_ack(); + } + } + } +} + void Connection::take_rtt_measure(const Packet& packet) { if(cb.SND.TS_OK) diff --git a/src/net/tcp/connection_states.cpp b/src/net/tcp/connection_states.cpp index f067116692..6ff844ba84 100644 --- a/src/net/tcp/connection_states.cpp +++ b/src/net/tcp/connection_states.cpp @@ -118,24 +118,25 @@ bool Connection::State::check_seq(Connection& tcp, const Packet& in) } debug2(" TCB: %s \n",tcb.to_string().c_str()); - // #1 + // #1 - The packet we expect if( in.seq() == tcb.RCV.NXT ) { goto acceptable; } - /// TODO: FIXME: reordering issues solved by this - else goto unacceptable; + /// if SACK isn't permitted there is no point handling out-of-order packets + else if(not tcp.sack_perm) + goto unacceptable; - // #2 + // #2 - Packet is ahead of what we expect to receive, but inside our window if( tcb.RCV.NXT <= in.seq() and in.seq() < tcb.RCV.NXT + tcb.RCV.WND ) { goto acceptable; } - // #3 (INVALID) + // #3 (INVALID) - Packet is outside the right edge of the recv window else if( in.seq() + in.tcp_data_length()-1 > tcb.RCV.NXT+tcb.RCV.WND ) { goto unacceptable; } - // #4 - else if( (tcb.RCV.NXT <= in.seq() and in.seq() < tcb.RCV.NXT + tcb.RCV.WND) - or ( tcb.RCV.NXT <= in.seq()+in.tcp_data_length()-1 and in.seq()+in.tcp_data_length()-1 < tcb.RCV.NXT+tcb.RCV.WND ) ) { + // #4 - Packet with payload is what we expect or bigger, but inside our window + else if( tcb.RCV.NXT <= in.seq()+in.tcp_data_length()-1 + and in.seq()+in.tcp_data_length()-1 < tcb.RCV.NXT+tcb.RCV.WND ) { goto acceptable; } /* @@ -303,106 +304,6 @@ bool Connection::State::check_ack(Connection& tcp, const Packet& in) { } } -///////////////////////////////////////////////////////////////////// -/* - 7. Process the segment text - - If a packet has data, process the data. - - [RFC 793] Page 74: - - Once in the ESTABLISHED state, it is possible to deliver segment - text to user RECEIVE buffers. Text from segments can be moved - into buffers until either the buffer is full or the segment is - empty. If the segment empties and carries an PUSH flag, then - the user is informed, when the buffer is returned, that a PUSH - has been received. - - When the TCP takes responsibility for delivering the data to the - user it must also acknowledge the receipt of the data. - - Once the TCP takes responsibility for the data it advances - RCV.NXT over the data accepted, and adjusts RCV.WND as - apporopriate to the current buffer availability. The total of - RCV.NXT and RCV.WND should not be reduced. - - Please note the window management suggestions in section 3.7. - - Send an acknowledgment of the form: - - - - This acknowledgment should be piggybacked on a segment being - transmitted if possible without incurring undue delay. -*/ -///////////////////////////////////////////////////////////////////// - -void Connection::State::process_segment(Connection& tcp, Packet& in) { - assert(in.has_tcp_data()); - - auto& tcb = tcp.tcb(); - auto length = in.tcp_data_length(); - // Receive could result in a user callback. This is used to avoid sending empty ACK reply. - debug(" Received packet with DATA-LENGTH: %i. Add to receive buffer. \n", length); - tcb.RCV.NXT += length; - const auto snd_nxt = tcb.SND.NXT; - if(tcp.read_request and tcp.read_request->buffer.capacity()) { - auto received = tcp.receive(in.seq(), in.tcp_data(), in.tcp_data_length(), in.isset(PSH)); - Ensures(received == length); - } - - // [RFC 5681] - //tcb.SND.cwnd += std::min(length, tcp.SMSS()); - debug2(" Advanced RCV.NXT: %u. SND.NXT = %u \n", tcb.RCV.NXT, snd_nxt); - - // user callback didnt result in transmitting an ACK - if (tcb.SND.NXT == snd_nxt) - { - // ACK by trying to send more - if (tcp.can_send()) - { - tcp.writeq_push(); - // nothing got sent - if (tcb.SND.NXT == snd_nxt) - { - tcp.send_ack(); - } - // something got sent - else - { - tcp.dack_ = 0; - } - } - // else regular ACK - else - { - if (tcp.use_dack() and tcp.dack_ == 0) - { - tcp.start_dack(); - //tcp.dack_ = 1; - } - else - { - tcp.stop_dack(); - tcp.send_ack(); - } - } - } - /* - WARNING/NOTE: - Not sure how "dangerous" the following is, and how big of a bottleneck it is. - Maybe has to be implemented with timers or something. - */ - - /* - Once the TCP takes responsibility for the data it advances - RCV.NXT over the data accepted, and adjusts RCV.WND as - apporopriate to the current buffer availability. The total of - RCV.NXT and RCV.WND should not be reduced. - */ -} -///////////////////////////////////////////////////////////////////// - ///////////////////////////////////////////////////////////////////// /* 8. Process FIN @@ -966,7 +867,7 @@ State::Result Connection::SynSent::handle(Connection& tcp, Packet_ptr in) { // 7. process segment text if(UNLIKELY(in->has_tcp_data())) { - process_segment(tcp, *in); + tcp.recv_data(*in); } // 8. check FIN bit @@ -985,7 +886,7 @@ State::Result Connection::SynSent::handle(Connection& tcp, Packet_ptr in) { tcp.transmit(std::move(packet)); tcp.set_state(Connection::SynReceived::instance()); if(in->has_tcp_data()) { - process_segment(tcp, *in); + tcp.recv_data(*in); } return OK; /* @@ -1063,7 +964,7 @@ State::Result Connection::SynReceived::handle(Connection& tcp, Packet_ptr in) { // 7. proccess the segment text if(UNLIKELY(in->has_tcp_data())) { - process_segment(tcp, *in); + tcp.recv_data(*in); } } /* @@ -1120,7 +1021,7 @@ State::Result Connection::Established::handle(Connection& tcp, Packet_ptr in) { // 7. proccess the segment text if(in->has_tcp_data()) { - process_segment(tcp, *in); + tcp.recv_data(*in); } // 8. check FIN bit @@ -1170,7 +1071,7 @@ State::Result Connection::FinWait1::handle(Connection& tcp, Packet_ptr in) { // 7. proccess the segment text if(in->has_tcp_data()) { - process_segment(tcp, *in); + tcp.recv_data(*in); } // 8. check FIN @@ -1221,7 +1122,7 @@ State::Result Connection::FinWait2::handle(Connection& tcp, Packet_ptr in) { // 7. proccess the segment text if(in->has_tcp_data()) { - process_segment(tcp, *in); + tcp.recv_data(*in); } // 8. check FIN From 151d9ec4ee0250b6fb852d4358693e55a481d5f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 30 Nov 2017 11:31:11 +0100 Subject: [PATCH 014/723] tcp: Moved ack of recv data to separate function --- api/net/tcp/connection.hpp | 7 +++++ src/net/tcp/connection.cpp | 52 +++++++++++++++++++++----------------- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index bbb1f5abd0..a1124cc664 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -1043,6 +1043,13 @@ class Connection { */ void recv_data(const Packet& in); + /** + * @brief Acknowledge incoming data. This is done by: + * - Trying to send data if possible (can send) + * - If not, regular ACK (use DACK if enabled) + */ + void ack_data(); + /** * @brief Determines if the incoming segment is a legit window update. * diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 5dfd01f1df..28605849b7 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -677,33 +677,39 @@ void Connection::recv_data(const Packet& in) // user callback didnt result in transmitting an ACK if (cb.SND.NXT == snd_nxt) { - // ACK by trying to send more - if (can_send()) + ack_data(); + } +} + +void Connection::ack_data() +{ + const auto snd_nxt = cb.SND.NXT; + // ACK by trying to send more + if (can_send()) + { + writeq_push(); + // nothing got sent + if (cb.SND.NXT == snd_nxt) { - writeq_push(); - // nothing got sent - if (cb.SND.NXT == snd_nxt) - { - send_ack(); - } - // something got sent - else - { - dack_ = 0; - } + send_ack(); } - // else regular ACK + // something got sent else { - if (use_dack() and dack_ == 0) - { - start_dack(); - } - else - { - stop_dack(); - send_ack(); - } + dack_ = 0; + } + } + // else regular ACK + else + { + if (use_dack() and dack_ == 0) + { + start_dack(); + } + else + { + stop_dack(); + send_ack(); } } } From af57e8f9c3cc27783fcc8be6269d796e65b9e2c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 30 Nov 2017 12:34:58 +0100 Subject: [PATCH 015/723] tcp: Make use of SACK in Connection (WIP) --- api/net/tcp/connection.hpp | 3 +++ api/net/tcp/sack.hpp | 22 +++++++++++----- api/util/fixed_storage.hpp | 2 +- src/net/tcp/connection.cpp | 52 ++++++++++++++++++++++++++++++-------- 4 files changed, 62 insertions(+), 17 deletions(-) diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index a1124cc664..a01155a29b 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -25,6 +25,7 @@ #include "rttm.hpp" #include "tcp_errors.hpp" #include "write_queue.hpp" +#include "sack.hpp" #include #include @@ -844,6 +845,8 @@ class Connection { /** State if connection is in TCP write queue or not. */ bool queued_; + using Sack_list = sack::List>; + std::unique_ptr sack_list; /** If SACK is permitted (option has been seen from peer) */ bool sack_perm = false; diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index 9a0fb2adb6..ecfb2267c4 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -14,12 +14,16 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +#pragma once +#ifndef NET_TCP_SACK_HPP +#define NET_TCP_SACK_HPP #include #include #include #include #include +#include using seq_t = uint32_t; @@ -80,7 +84,7 @@ struct Block { }__attribute__((packed)); // Print for Block -std::ostream& operator<<(std::ostream& out, const Block& b) { +inline std::ostream& operator<<(std::ostream& out, const Block& b) { out << "[" << b.start << " => " << b.end << "]"; return out; } @@ -107,6 +111,9 @@ class List { Ack_result new_valid_ack(const seq_t end) { return impl.new_valid_ack(end); } + size_t size() const noexcept + { return impl.size(); } + List_impl impl; }; @@ -142,7 +149,7 @@ connects_to(Iterator first, Iterator last, const Connectable& value) template class Fixed_list { public: - static auto constexpr size = N; + static auto constexpr capacity = N; using List = std::list>; using List_iterator = typename List::iterator; @@ -172,9 +179,9 @@ class Fixed_list { } else // No connection - new entry { - Expects(blocks.size() <= size); + Expects(blocks.size() <= capacity); - if(UNLIKELY(blocks.size() == size)) + if(UNLIKELY(blocks.size() == capacity)) return {recent_entries(), 0}; blocks.push_front(blk); @@ -200,6 +207,9 @@ class Fixed_list { return {recent_entries(), bytes_freed}; } + size_t size() const noexcept + { return blocks.size(); } + void move_to_front(List_iterator it) { if(it != blocks.begin()) @@ -209,8 +219,6 @@ class Fixed_list { Entries recent_entries() const { Entries ret; - int i = 0; - for(auto it = blocks.begin(); it != blocks.end() and ret.size() < ret.capacity(); it++) ret.push_back(*it); @@ -308,3 +316,5 @@ class Scoreboard_list { } // < namespace sack } // < namespace tcp } // < namespace net + +#endif diff --git a/api/util/fixed_storage.hpp b/api/util/fixed_storage.hpp index e113b798c3..1d9c97844b 100644 --- a/api/util/fixed_storage.hpp +++ b/api/util/fixed_storage.hpp @@ -106,7 +106,7 @@ char* Fixed_storage::allocate(std::size_t n) } template -void Fixed_storage::deallocate(char* p, std::size_t n) noexcept +void Fixed_storage::deallocate(char* p, std::size_t) noexcept { Expects(pointer_in_buffer(p) && "Trying to deallocate pointer outside my buffer"); diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 28605849b7..a7c9fae549 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -663,22 +663,54 @@ void Connection::rtx_ack(const seq_t ack) { void Connection::recv_data(const Packet& in) { Expects(in.has_tcp_data()); + const auto length = in.tcp_data_length(); - cb.RCV.NXT += length; - const auto snd_nxt = cb.SND.NXT; - if(read_request and read_request->buffer.capacity()) { - auto received = receive(in.seq(), in.tcp_data(), in.tcp_data_length(), in.isset(PSH)); - Ensures(received == length); + + // The packet we expect + if(cb.RCV.NXT == in.seq()) + { + // If we had packet loss before (and SACK is on) + // we need to clear up among the blocks + // and increase the total amount of bytes acked + if(UNLIKELY(sack_list)) + { + auto res = sack_list->new_valid_ack(in.seq()); + cb.RCV.NXT += res.bytes; + } + + cb.RCV.NXT += length; + } + // Packet out of order + else + { + // We know that out of order packets wouldnt reach here + // without SACK being permitted + Expects(sack_perm); + + printf(" Out-of-order: RCV.NXT=%u SEQ=%u\n", + cb.RCV.NXT, in.seq()); + + // The SACK list is initated on the first out of order packet + if(not sack_list) + sack_list = std::make_unique(); + + auto res = sack_list->recv_out_of_order(in.seq(), length); } - // [RFC 5681] - //tcb.SND.cwnd += std::min(length, tcp.SMSS()); + // Keep track if a packet is being sent during the async read callback + const auto snd_nxt = cb.SND.NXT; - // user callback didnt result in transmitting an ACK - if (cb.SND.NXT == snd_nxt) + if(read_request and read_request->buffer.capacity()) { - ack_data(); + auto recv = receive(in.seq(), in.tcp_data(), length, in.isset(PSH)); + Ensures(recv == length); } + + // User callback didnt result in transmitting an ACK + if(cb.SND.NXT == snd_nxt) + ack_data(); + + // [RFC 5681] ??? } void Connection::ack_data() From 0be209047df519eebb67078293af438da38898f7 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 12 Dec 2017 19:21:31 +0100 Subject: [PATCH 016/723] WIP: replacing newlib with musl --- CMakeLists.txt | 3 +- api/kernel/syscalls.hpp | 1 - api/posix/README.md | 11 - api/posix/arpa/inet.h | 82 - api/posix/cpio.h | 51 - api/posix/ctype.h | 45 - api/posix/dlfcn.h | 47 - api/posix/features.h | 41 - api/posix/ftw.h | 37 - api/posix/inet.h | 27 - api/posix/info.h | 40 - api/posix/locale.h | 25 - api/posix/math.h | 101 - api/posix/netdb.h | 123 -- api/posix/netinet/in.h | 83 - api/posix/netinet/tcp.h | 31 - api/posix/nl_types.h | 39 - api/posix/posix_strace.hpp | 9 - api/posix/pthread.h | 91 - api/posix/sched.h | 25 - api/posix/signal.h | 42 - api/posix/stddef.h | 45 +- api/posix/stdlib.h | 37 - api/posix/sys/__config | 4 - api/posix/sys/ioctl.h | 11 - api/posix/sys/mman.h | 79 - api/posix/sys/socket.h | 156 -- api/posix/sys/uio.h | 38 - api/posix/sys/utsname.h | 52 - api/posix/syslog.h | 155 -- api/posix/tar.h | 94 - api/posix/time.h | 25 - api/posix/ucontext.h | 62 - api/posix/unistd.h | 35 - api/posix/wchar.h | 36 - api/util/elf.h | 2813 ---------------------------- api/util/tar.hpp | 2 +- cmake/cross_compiled_libraries.txt | 12 +- etc/build_llvm.sh | 31 +- etc/build_musl.sh | 22 + etc/create_binary_bundle.sh | 60 +- lib/LiveUpdate/CMakeLists.txt | 2 +- lib/mana/CMakeLists.txt | 2 +- lib/mender/CMakeLists.txt | 2 +- lib/microLB/CMakeLists.txt | 2 +- lib/uplink/CMakeLists.txt | 2 +- src/CMakeLists.txt | 9 +- src/kernel/elf.cpp | 2 +- src/posix/arpa/inet.cpp | 115 -- src/posix/dirent.cpp | 40 - src/posix/dlfcn.cpp | 57 - src/posix/fcntl.cpp | 56 - src/posix/fd.cpp | 60 - src/posix/file_fd.cpp | 59 - src/posix/ftw.cpp | 99 - src/posix/math.cpp | 6 - src/posix/poll.cpp | 46 - src/posix/pthread.cpp | 188 -- src/posix/pwd.cpp | 168 -- src/posix/signal.cpp | 16 - src/posix/stdio.cpp | 71 - src/posix/sys/ioctl.cpp | 7 - src/posix/sys/mman.cpp | 89 - src/posix/sys/select.cpp | 168 -- src/posix/sys/socket.cpp | 186 -- src/posix/sys/stat.cpp | 307 --- src/posix/sys/utsname.cpp | 38 - src/posix/syslog.cpp | 144 -- src/posix/tcp_fd.cpp | 384 ---- src/posix/termios.cpp | 27 - src/posix/ucontext.cpp | 103 - src/posix/ucontext_asm.asm | 163 -- src/posix/udp_fd.cpp | 389 ---- src/posix/unistd.cpp | 480 ----- src/posix/utime.cpp | 18 - 75 files changed, 78 insertions(+), 8150 deletions(-) delete mode 100644 api/posix/README.md delete mode 100644 api/posix/arpa/inet.h delete mode 100644 api/posix/cpio.h delete mode 100644 api/posix/ctype.h delete mode 100644 api/posix/dlfcn.h delete mode 100644 api/posix/features.h delete mode 100644 api/posix/ftw.h delete mode 100644 api/posix/inet.h delete mode 100644 api/posix/info.h delete mode 100644 api/posix/locale.h delete mode 100644 api/posix/math.h delete mode 100644 api/posix/netdb.h delete mode 100644 api/posix/netinet/in.h delete mode 100644 api/posix/netinet/tcp.h delete mode 100644 api/posix/nl_types.h delete mode 100644 api/posix/posix_strace.hpp delete mode 100644 api/posix/pthread.h delete mode 100644 api/posix/sched.h delete mode 100644 api/posix/signal.h delete mode 100644 api/posix/stdlib.h delete mode 100644 api/posix/sys/__config delete mode 100644 api/posix/sys/ioctl.h delete mode 100644 api/posix/sys/mman.h delete mode 100644 api/posix/sys/socket.h delete mode 100644 api/posix/sys/uio.h delete mode 100644 api/posix/sys/utsname.h delete mode 100644 api/posix/syslog.h delete mode 100644 api/posix/tar.h delete mode 100644 api/posix/time.h delete mode 100644 api/posix/ucontext.h delete mode 100644 api/posix/unistd.h delete mode 100644 api/posix/wchar.h delete mode 100644 api/util/elf.h create mode 100755 etc/build_musl.sh delete mode 100644 src/posix/arpa/inet.cpp delete mode 100644 src/posix/dirent.cpp delete mode 100644 src/posix/dlfcn.cpp delete mode 100644 src/posix/fcntl.cpp delete mode 100644 src/posix/fd.cpp delete mode 100644 src/posix/file_fd.cpp delete mode 100644 src/posix/ftw.cpp delete mode 100644 src/posix/math.cpp delete mode 100644 src/posix/poll.cpp delete mode 100644 src/posix/pthread.cpp delete mode 100644 src/posix/pwd.cpp delete mode 100644 src/posix/signal.cpp delete mode 100644 src/posix/stdio.cpp delete mode 100644 src/posix/sys/ioctl.cpp delete mode 100644 src/posix/sys/mman.cpp delete mode 100644 src/posix/sys/select.cpp delete mode 100644 src/posix/sys/socket.cpp delete mode 100644 src/posix/sys/stat.cpp delete mode 100644 src/posix/sys/utsname.cpp delete mode 100644 src/posix/syslog.cpp delete mode 100644 src/posix/tcp_fd.cpp delete mode 100644 src/posix/termios.cpp delete mode 100644 src/posix/ucontext.cpp delete mode 100644 src/posix/ucontext_asm.asm delete mode 100644 src/posix/udp_fd.cpp delete mode 100644 src/posix/unistd.cpp delete mode 100644 src/posix/utime.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1121143a77..65f00634fe 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,8 +62,7 @@ set(CAPABS "${CAPABS} -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_P # Various global defines # * NO_DEBUG disables output from the debug macro # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures -# * _GNU_SOURCE enables POSIX-extensions in newlib, such as strnlen. ("everything newlib has", ref. cdefs.h) -set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_GNU_SOURCE") +set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE") set(WARNS "-Wall -Wextra") diff --git a/api/kernel/syscalls.hpp b/api/kernel/syscalls.hpp index 143929dc2b..9a99bc8182 100644 --- a/api/kernel/syscalls.hpp +++ b/api/kernel/syscalls.hpp @@ -18,7 +18,6 @@ #ifndef KERNEL_SYSCALLS_HPP #define KERNEL_SYSCALLS_HPP -#include #include extern "C" { diff --git a/api/posix/README.md b/api/posix/README.md deleted file mode 100644 index c785e802d7..0000000000 --- a/api/posix/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# IncludeOS POSIX extensions - -IncludeOS intends to provide a POSIX interface sufficient for linking and running many conventional C libraries and programs. A lot of the POSIX functionality will be header-only stubs and some of it is provided externally by e.g. the compiler or standard library. - -### Other providers of POSIX content -* *newlib*: is our current C library which also provides many POSIX features (indeed the C standard itself overlaps with POSIX). Newlib provides most of the C standard library, including `stdlib.h`, `stdio.h`, `math.h` etc., but is missing some C11 extensions. Those are rarely used and provided here as stubs. -* *clang*: Clang provides a few POSIX headers such as `stddef.h`, `stdarg.h` and `limits.h`. It also provides compiler intrinsics such as `x86intrin.h`. When building IncludeOS we use the `-nostdlibinc` flag to allow inclusion of these headers, without including the standard library headers from the host. - -### Guidelines for this folder -* Only actual standardized POSIX content should live here, and only content not already provided by alternative sources above. -* Extensions to POSIX headers that IncludeOS needs, but which isn't present on one of the supported platforms (e.g. macOS or Linux) should not live here, since we'd like to be able to build directly on those platforms with their respective POSIX implementations. As an example, our syslog implementation defines `LOG_INTERNAL` in addition to `LOG_MAIL` etc. While defining this symbol in the `syslog.h` POSIX header is allowed by the standard it introduces an implicit expectation in IncludeOS application code making it less portable. Such extensions can be placed in the IncludeOS API instead. diff --git a/api/posix/arpa/inet.h b/api/posix/arpa/inet.h deleted file mode 100644 index 8cae384f68..0000000000 --- a/api/posix/arpa/inet.h +++ /dev/null @@ -1,82 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef POSIX_ARPA_INET_H -#define POSIX_ARPA_INET_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "../netinet/in.h" -#include - -#ifdef ntohs -#undef ntohs -#endif -#ifdef htons -#undef htons -#endif -#ifdef ntohl -#undef ntohl -#endif -#ifdef htonl -#undef htonl -#endif -#ifdef ntohll -#undef ntohll -#endif -#ifdef htonll -#undef htonll -#endif - -inline uint16_t ntohs(const uint16_t n) { - return __builtin_bswap16(n); -} - -inline uint16_t htons(const uint16_t n) { - return __builtin_bswap16(n); -} - -inline uint32_t ntohl(const uint32_t n) { - return __builtin_bswap32(n); -} - -inline uint32_t htonl(const uint32_t n) { - return __builtin_bswap32(n); -} - -inline uint64_t ntohll(const uint64_t n) { - return __builtin_bswap64(n); -} - -inline uint64_t htonll(const uint64_t n) { - return __builtin_bswap64(n); -} - -in_addr_t inet_addr(const char *); -char *inet_ntoa(struct in_addr); -const char *inet_ntop(int, const void *__restrict__, char *__restrict__, - socklen_t); -int inet_pton(int, const char *__restrict__, void *__restrict__); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/api/posix/cpio.h b/api/posix/cpio.h deleted file mode 100644 index 6602ef395d..0000000000 --- a/api/posix/cpio.h +++ /dev/null @@ -1,51 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef POSIX_CPIO_H -#define POSIX_CPIO_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define C_IRUSR 0000400 -#define C_IWUSR 0000200 -#define C_IXUSR 0000100 -#define C_IRGRP 0000040 -#define C_IWGRP 0000020 -#define C_IXGRP 0000010 -#define C_IROTH 0000004 -#define C_IWOTH 0000002 -#define C_IXOTH 0000001 -#define C_ISUID 0004000 -#define C_ISGID 0002000 -#define C_ISVTX 0001000 -#define C_ISDIR 0040000 -#define C_ISFIFO 0010000 -#define C_ISREG 0100000 -#define C_ISBLK 0060000 -#define C_ISCHR 0020000 -#define C_ISCTG 0110000 -#define C_ISLNK 0120000 -#define C_ISSOCK 0140000 - -#define MAGIC "070707" - -#ifdef __cplusplus -} -#endif -#endif diff --git a/api/posix/ctype.h b/api/posix/ctype.h deleted file mode 100644 index 11ccc683a4..0000000000 --- a/api/posix/ctype.h +++ /dev/null @@ -1,45 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef SYS_CTYPE_H -#define SYS_CTYPE_H - -#define _LIBCPP_SUPPORT_XLOCALE_POSIX_L_FALLBACK_H - -#ifdef __cplusplus -extern "C" { -#endif - - static inline int isascii(int c){ - return c >= 0 && c <= 0177; - } - - static inline int isalnum(int ch){ - // http://en.cppreference.com/w/cpp/string/byte/isalnum - return (ch >= 48 && ch <= 57) - || (ch >= 65 && ch <= 90) - || (ch >= 97 && ch <= 122); - } - - -#ifdef __cplusplus -} -#endif - -#endif //SYS_CTYPE_H - -#include_next diff --git a/api/posix/dlfcn.h b/api/posix/dlfcn.h deleted file mode 100644 index 8456d0ae1b..0000000000 --- a/api/posix/dlfcn.h +++ /dev/null @@ -1,47 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef POSIX_DLFCN_H -#define POSIX_DLFCN_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define RTLD_LAZY 1 // Relocations are performed at an implementation-dependent time. -#define RTLD_NOW 2 // Relocations are performed when the object is loaded. -#define RTLD_GLOBAL 3 // All symbols are available for relocation processing of other modules. -#define RTLD_LOCAL 4 // All symbols are not made available for relocation processing by other modules - -// Extra linux-specific defines -#define RTLD_NODELETE 6 -#define RTLD_NOLOAD 7 -#define RTLD_DEEPBIND 8 - -#define RTLD_DEFAULT 1 - - -void *dlopen(const char *, int); -void *dlsym(void *, const char *); -int dlclose(void *); -char *dlerror(void); - -#ifdef __cplusplus -} -#endif - -#endif // < POSIX_DLFCN_H diff --git a/api/posix/features.h b/api/posix/features.h deleted file mode 100644 index 0fc2a44f44..0000000000 --- a/api/posix/features.h +++ /dev/null @@ -1,41 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef FEATURES_H -#define FEATURES_H - -/** - * Enabling / disabling POSIX features - * Note: libc++ uses , newlib - **/ - -// Newlib needs this switch to enable clock_gettime etc. -#define _POSIX_TIMERS 1 -#define _POSIX_MONOTONIC_CLOCK 1 - -// Enable newlib multibyte support -#define _MB_CAPABLE 1 - - - -// Required to pass CMake tests for libc++ -#define __GLIBC_PREREQ__(min, maj) 1 -#define __GLIBC_PREREQ(min, maj) 1 - -#include_next - -#endif diff --git a/api/posix/ftw.h b/api/posix/ftw.h deleted file mode 100644 index 84352c6991..0000000000 --- a/api/posix/ftw.h +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef FTW_H -#define FTW_H - -#include - -#ifdef __cplusplus -extern "C" { -#endif - -struct FTW { - int base; - int level; -}; - -#define FTW_F 0 // Non-directory file. -#define FTW_D 1 // Directory. -#define FTW_DNR 2 // Directory without read permission. -#define FTW_DP 3 // Directory with subdirectories visited. -#define FTW_NS 4 // Unknown type; stat() failed. -#define FTW_SL 5 // Symbolic link. -#define FTW_SLN 6 // Symbolic link that names a nonexistent file. - - -#define FTW_PHYS 0x01 -#define FTW_MOUNT 0x02 -#define FTW_DEPTH 0x04 -#define FTW_CHDIR 0x08 - -typedef int Nftw_func(const char *, const struct stat *, int, struct FTW *); -//int nftw(const char *, int (*)(const char *, const struct stat *, int, struct FTW *), int, int); -int nftw(const char *path, Nftw_func fn, int fd_limit, int flags); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/api/posix/inet.h b/api/posix/inet.h deleted file mode 100644 index 81da9665f3..0000000000 --- a/api/posix/inet.h +++ /dev/null @@ -1,27 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef POSIX_INET_H -#define POSIX_INET_H - -#define INADDR_ANY 0x00000000 -#define INADDR_LOOPBACK 0x7f000001 -#define INADDR_NONE 0xffffffff -#define INPORT_ANY 0 - -#endif diff --git a/api/posix/info.h b/api/posix/info.h deleted file mode 100644 index 7022a62fc0..0000000000 --- a/api/posix/info.h +++ /dev/null @@ -1,40 +0,0 @@ -// -*-C++-*- -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef API_POSIX_INFO_H -#define API_POSIX_INFO_H - -#define LINEWIDTH 80 - -#ifndef NO_INFO -#define INFO(FROM, TEXT, ...) printf("%13s ] " TEXT "\n", "[ " FROM, ##__VA_ARGS__) -#define INFO2(TEXT, ...) printf("%16s" TEXT "\n"," ", ##__VA_ARGS__) -#define CAPTION(TEXT) printf("\n%*s%*s\n",LINEWIDTH/2 + strlen(TEXT)/2,TEXT,LINEWIDTH/2-strlen(TEXT)/2,"") -#define CHECK(TEST, TEXT, ...) printf("%16s[%s] " TEXT "\n","", TEST ? "x" : " ", ##__VA_ARGS__) -#define CHECKSERT(TEST, TEXT, ...) assert(TEST), CHECK(TEST, TEXT, ##__VA_ARGS__) - -#else -#define INFO(X,...) -#define INFO2(X,...) -#define CAPTION(TEXT,...) -#define FILLINE(CHAR) -#define CHECK(X,...) -#endif - -#endif //< ___API_INFO___ diff --git a/api/posix/locale.h b/api/posix/locale.h deleted file mode 100644 index 641bc90fa5..0000000000 --- a/api/posix/locale.h +++ /dev/null @@ -1,25 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef SYS_LOCALE_H -#define SYS_LOCALE_H - -#define _LIBCPP_SUPPORT_XLOCALE_NOP_LOCALE_MGMT_H - -#include_next - -#endif diff --git a/api/posix/math.h b/api/posix/math.h deleted file mode 100644 index fe0c8bbec3..0000000000 --- a/api/posix/math.h +++ /dev/null @@ -1,101 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef SYS_MATH_H -#define SYS_MATH_H - -#ifdef __cplusplus -extern "C" { -#endif - -// Long double math stubs -long double cosl(long double); -long double sinl(long double); -long double tanl(long double); - -long double acosl(long double); -long double asinl(long double); -long double atanl(long double); -long double atan2l(long double, long double); - - -long double coshl(long double); -long double sinhl(long double); -long double tanhl(long double); -long double acoshl(long double); -long double asinhl(long double); -long double atanhl(long double); - - -long double ceill(long double); -long double coshl(long double); -long double fabsl(long double); -long double floorl(long double); -long double fmodl(long double, long double); -long double logl(long double); -long double log10l(long double); -long double modfl(long double, long double*); -long double powl(long double,long double); -long double frexpl(long double, int*); - -long double cbrtl(long double); -long double copysignl(long double, long double); -long double erfl(long double); -long double erfcl(long double); -long double exp2l(long double); -long double expm1l(long double); -long double fdiml(long double, long double); -long double fmal(long double, long double, long double); -long double fmaxl(long double, long double); -long double fminl(long double, long double); - -long double expl(long double); -long double ldexpl(long double, long double); - -long double ilogbl(long double); -long double lgammal(long double); -long double llroundl(long double); -long double log1pl(long double); -long double log2l(long double); -long double logbl(long double); -long double lroundl(long double); -long double nearbyintl(long double); -long double nextafterl(long double, long double); -double nexttoward(double, long double); -long double nexttowardl(long double, long double); -long double nexttowardf(long double, long double); - -long double remainderl(long double, long double); -long double remquol(long double, long double, int*); -long double roundl(long double); -long double scalblnl(long double, long double); -long double scalbnl(long double, long double); -long double tgammal(long double); -long double truncl(long double); -long double nanl(const char*); - -long double rintl(long double); -long lrintl(long double); -long long llrintl(long double); - -#ifdef __cplusplus -} -#endif - -#include_next - -#endif diff --git a/api/posix/netdb.h b/api/posix/netdb.h deleted file mode 100644 index 0e73f4b120..0000000000 --- a/api/posix/netdb.h +++ /dev/null @@ -1,123 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef POSIX_NETDB_H -#define POSIX_NETDB_H - -#ifdef __cplusplus -extern "C" { -#endif - -struct hostent { - char *h_name; // Official name of the host - char **h_aliases; // Pointer to array of pointers to alternative host names - int h_addrtype; // Address type - int h_length; // Length of address - char **h_addr_list; // Pointer to array of pointers to network addresses -}; - -struct netent { - char *n_name; // FQDN of host - char **n_aliases; // Pointer to array of pointers to alternative network names - int n_addrtype; // Address type - uint32_t n_net; // Network number -}; - -struct protoent { - char *p_name; // Name of protocol - char **p_aliases; // Pointer to array of pointers to alternative protocol names - int p_proto; // Protocol number -}; - -struct servent { - char *s_name; // Name of service - char **s_aliases; // Pointer to array of pointers to alternative service names - int s_port; // Port number - char *s_proto; //Name of protocol -}; - -struct addrinfo { - int ai_flags; // Input flags - int ai_family; // Address family - int ai_socktype; // Socket type - int ai_protocol; // Protocol - socklen_t ai_addrlen; // Length of socket address - struct sockaddr *ai_addr; // Socket address - char *ai_canonname; // Canonical name of service location - struct addrinfo *ai_next; // Pointer to next in list -}; - -// Constants for use in flags field of addrinfo structure -#define AI_PASSIVE 1 -#define AI_CANONNAME 2 -#define AI_NUMERICHOST 4 -#define AI_NUMERICSERV 8 -#define AI_V4MAPPED 16 -#define AI_ALL 32 -#define AI_ADDRCONFIG 64 - -// Constants for use in flags argument to getnameinfo() -#define NI_NOFQDN 1 -#define NI_NUMERICHOST 2 -#define NI_NAMEREQD 4 -#define NI_NUMERICSERV 8 -#define NI_NUMERICSCOPE 16 -#define NI_DGRAM 32 - -// Constants for error values for getaddrinfo() and getnameinfo() -#define EAI_AGAIN -1 -#define EAI_BADFLAGS -2 -#define EAI_FAIL -3 -#define EAI_FAMILY -4 -#define EAI_MEMORY -5 -#define EAI_NONAME -6 -#define EAI_SERVICE -7 -#define EAI_SOCKTYPE -8 -#define EAI_SYSTEM -9 -#define EAI_OVERFLOW -10 - -void endhostent(void); -void endnetent(void); -void endprotoent(void); -void endservent(void); -void freeaddrinfo(struct addrinfo *); -const char *gai_strerror(int); -int getaddrinfo(const char *__restrict__, const char *__restrict__, - const struct addrinfo *__restrict__, struct addrinfo **__restrict__); -struct hostent *gethostbyname(const char *name); -struct hostent *gethostent(void); -int getnameinfo(const struct sockaddr *__restrict__, socklen_t, char *__restrict__, - socklen_t, char *__restrict__, socklen_t, int); -struct netent *getnetbyaddr(uint32_t, int); -struct netent *getnetbyname(const char *); -struct netent *getnetent(void); -struct protoent *getprotobyname(const char *); -struct protoent *getprotobynumber(int); -struct protoent *getprotoent(void); -struct servent *getservbyname(const char *, const char *); -struct servent*getservbyport(int, const char *); -struct servent *getservent(void); -void sethostent(int); -void setnetent(int); -void setprotoent(int); -void setservent(int); - -#ifdef __cplusplus -} -#endif - -#endif // POSIX_NETDB_H diff --git a/api/posix/netinet/in.h b/api/posix/netinet/in.h deleted file mode 100644 index 376ea06f6e..0000000000 --- a/api/posix/netinet/in.h +++ /dev/null @@ -1,83 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef POSIX_NETINET_IN_H -#define POSIX_NETINET_IN_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -typedef uint16_t in_port_t; -typedef uint32_t in_addr_t; - -struct in_addr -{ - in_addr_t s_addr; -}; -struct sockaddr_in -{ - sa_family_t sin_family; - in_port_t sin_port; - struct in_addr sin_addr; - char sin_zero[8]; -}; - - -struct in6_addr -{ - uint8_t s6_addr[16]; -}; -struct sockaddr_in6 -{ - sa_family_t sin6_family; - uint16_t sin6_port; - uint32_t sin6_flowinfo; - struct in6_addr sin6_addr; - uint32_t sin6_scope_id; -}; - -extern const struct in6_addr in6addr_any; -extern const struct in6_addr in6addr_loopback; - -/// fixme: use actual inet protocol values -#define IPPROTO_IP 0 -#define IPPROTO_ICMP 1 -#define IPPROTO_TCP 6 -#define IPPROTO_UDP 17 -#define IPPROTO_IPV6 41 -#define IPPROTO_RAW 255 - -#define INADDR_ANY 0x0 -#define INADDR_BROADCAST 0xffffffff -#define INADDR_NONE 0xffffffff -#define INADDR_LOOPBACK 0x7f000001 - -#define INET_ADDRSTRLEN 16 -#define INET6_ADDRSTRLEN 46 - -#include "../arpa/inet.h" - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/api/posix/netinet/tcp.h b/api/posix/netinet/tcp.h deleted file mode 100644 index d78dd6558d..0000000000 --- a/api/posix/netinet/tcp.h +++ /dev/null @@ -1,31 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef POSIX_NETINET_TCP_H -#define POSIX_NETINET_TCP_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define TCP_NODELAY 1 // Avoid coalescing of small segments - -#ifdef __cplusplus -} -#endif - -#endif // POSIX_NETINET_TCP_H diff --git a/api/posix/nl_types.h b/api/posix/nl_types.h deleted file mode 100644 index 5660e4291c..0000000000 --- a/api/posix/nl_types.h +++ /dev/null @@ -1,39 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef POSIX_NL_TYPES_H -#define POSIX_NL_TYPES_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define NL_SETD 1 -#define NL_CAT_LOCALE 1 - -typedef void *nl_catd; -typedef int nl_item; - -int catclose(nl_catd); -char *catgets(nl_catd, int, int, const char *); -nl_catd catopen(const char *, int); - -#ifdef __cplusplus -} -#endif - -#endif // < POSIX_NL_TYPES_H diff --git a/api/posix/posix_strace.hpp b/api/posix/posix_strace.hpp deleted file mode 100644 index 866ad0036c..0000000000 --- a/api/posix/posix_strace.hpp +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once -#include - -//#define POSIX_STRACE -#ifdef POSIX_STRACE -#define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) -#else -#define PRINT(fmt, ...) /* fmt */ -#endif diff --git a/api/posix/pthread.h b/api/posix/pthread.h deleted file mode 100644 index f58e5e0d2c..0000000000 --- a/api/posix/pthread.h +++ /dev/null @@ -1,91 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include - -extern "C" -{ -typedef unsigned int pthread_t; -typedef unsigned int pthread_key_t; -typedef struct { - int is_initialized; - void *stackaddr; - int stacksize; - int contentionscope; - int inheritsched; - int schedpolicy; - sched_param schedparam; - int detachstate; -} pthread_attr_t; - - -typedef struct {} pthread_barrier_t; -typedef struct {} pthread_barrierattr_t; -typedef struct {} pthread_cond_t; -typedef struct {} pthread_condattr_t; -typedef struct {} pthread_mutexattr_t; -typedef struct {} pthread_rwlock_t; -typedef struct {} pthread_rwlockattr_t; -typedef struct {} pthread_spinlock_t; - -typedef volatile int spinlock_t __attribute__((aligned(128))); -typedef struct { - spinlock_t spinlock; -} pthread_mutex_t; - -int pthread_cond_broadcast(pthread_cond_t* cond); -int pthread_cond_signal(pthread_cond_t* cond); -int pthread_cond_timedwait(pthread_cond_t* cond, - pthread_mutex_t* mutex, - const struct timespec* abstime); -int pthread_cond_wait(pthread_cond_t* cond, - pthread_mutex_t* mutex); -int pthread_cond_destroy(pthread_cond_t *cond); -int pthread_cond_init(pthread_cond_t *__restrict cond, - const pthread_condattr_t *__restrict attr); - -pthread_t pthread_self(); -int pthread_create(pthread_t *, const pthread_attr_t *, void *(*)(void *), void *); -int pthread_join(pthread_t, void **); -int pthread_detach(pthread_t thread); -int pthread_equal(pthread_t t1, pthread_t t2); - -void *pthread_getspecific(pthread_key_t key); -int pthread_setspecific(pthread_key_t key, const void *value); -int pthread_key_create(pthread_key_t *key, void (*destructor)(void*)); -int pthread_mutexattr_destroy(pthread_mutexattr_t *attr); -int pthread_mutexattr_init(pthread_mutexattr_t *attr); -int pthread_mutexattr_gettype(const pthread_mutexattr_t *__restrict attr, int *__restrict type); -int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type); - -typedef struct {} pthread_once_t; -int pthread_once(pthread_once_t *once_control, void (*init_routine)(void)); - -#define PTHREAD_MUTEX_INITIALIZER {0} -#define PTHREAD_COND_INITIALIZER {} -#define PTHREAD_ONCE_INIT {} - -#define PTHREAD_MUTEX_RECURSIVE 33 - -int pthread_mutex_init(pthread_mutex_t* mutex, const pthread_mutexattr_t* attr); -int pthread_mutex_lock(pthread_mutex_t *mutex); -int pthread_mutex_trylock(pthread_mutex_t *mutex); -int pthread_mutex_unlock(pthread_mutex_t *mutex); -int pthread_mutex_destroy(pthread_mutex_t *mutex); - -} // "C" diff --git a/api/posix/sched.h b/api/posix/sched.h deleted file mode 100644 index cecf6a670a..0000000000 --- a/api/posix/sched.h +++ /dev/null @@ -1,25 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include - -typedef struct { - int sched_priority; -} sched_param; - -extern int sched_yield(); diff --git a/api/posix/signal.h b/api/posix/signal.h deleted file mode 100644 index 42c677b159..0000000000 --- a/api/posix/signal.h +++ /dev/null @@ -1,42 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef SYS_SIGNAL_H -#define SYS_SIGNAL_H - -#include_next - -#ifdef __cplusplus -extern "C" { -#endif - -struct siginfo_t -{ - int si_signo; // Signal number. - int si_code; // Signal code. - - int sa_sigaction; -}; - -#define SA_RESETHAND 666 - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/api/posix/stddef.h b/api/posix/stddef.h index 553a8e0412..3044a26246 100644 --- a/api/posix/stddef.h +++ b/api/posix/stddef.h @@ -1,43 +1,2 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef SYS_STDDEF_H -#define SYS_STDDEF_H -//#if defined(__need_ptrdiff_t) || defined(__need_size_t) || defined(__need_wchar_t) || defined(__need_NULL) || defined(__need_wint_t) - -typedef __PTRDIFF_TYPE__ ptrdiff_t; -typedef __SIZE_TYPE__ size_t; - -#ifndef __cplusplus -typedef __WCHAR_TYPE__ wchar_t; -#endif -typedef __WINT_TYPE__ wint_t; - -#ifndef NULL -#ifdef __cplusplus - #define NULL __null -#else - #define NULL ((void*) 0) -#endif -#endif // NULL - -#ifndef offsetof -#define offsetof(t, d) __builtin_offsetof(t, d) -#endif - -#endif +#define _GCC_MAX_ALIGN_T +#include_next diff --git a/api/posix/stdlib.h b/api/posix/stdlib.h deleted file mode 100644 index d7e1cf388b..0000000000 --- a/api/posix/stdlib.h +++ /dev/null @@ -1,37 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Our patches / additions to newlibs partial implementation -#ifndef SYS_STDLIB_H -#define SYS_STDLIB_H - -#include_next - -#ifdef __cplusplus -extern "C" { -#endif - - //New stuff in C11, required by libunwind, compiler-rt etc. in llvm - void *aligned_alloc( size_t alignment, size_t size ); - int at_quick_exit (void (*func)(void)); - _Noreturn void quick_exit (int status); - - -#ifdef __cplusplus -} -#endif -#endif //SYS_STDLIB_H diff --git a/api/posix/sys/__config b/api/posix/sys/__config deleted file mode 100644 index dd551fe023..0000000000 --- a/api/posix/sys/__config +++ /dev/null @@ -1,4 +0,0 @@ -// -*- C++ -*- -// This breaks libc++ build, but is required to build seed. -//#define _LIBCPP_HAS_NO_THREADS 1 -#include_next <__config> diff --git a/api/posix/sys/ioctl.h b/api/posix/sys/ioctl.h deleted file mode 100644 index 7722a9588b..0000000000 --- a/api/posix/sys/ioctl.h +++ /dev/null @@ -1,11 +0,0 @@ -#pragma once - -#ifdef __cplusplus -extern "C" { -#endif - -int ioctl(int fd, unsigned long request, ...); - -#ifdef __cplusplus -} -#endif diff --git a/api/posix/sys/mman.h b/api/posix/sys/mman.h deleted file mode 100644 index 2b8f7daecf..0000000000 --- a/api/posix/sys/mman.h +++ /dev/null @@ -1,79 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef POSIX_SYS_MMAN_H -#define POSIX_SYS_MMAN_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include -typedef _off_t off_t; - -struct posix_typed_mem_info -{ - size_t posix_tmi_length; -}; - -int mlock(const void *, size_t); -int mlockall(int); -void *mmap(void *, size_t, int, int, int, off_t); -void *mremap(void*, size_t, size_t, int flags, ... /* void *new_address */); -int mprotect(void *, size_t, int); -int msync(void *, size_t, int); -int munlock(const void *, size_t); -int munlockall(void); -int munmap(void *, size_t); -int posix_madvise(void *, size_t, int); -int posix_mem_offset(const void *__restrict__, size_t, off_t *__restrict__, - size_t *__restrict__, int *__restrict__); -int posix_typed_mem_get_info(int, struct posix_typed_mem_info *); -int posix_typed_mem_open(const char *, int, int); -int shm_open(const char *, int, mode_t); -int shm_unlink(const char *); - -// Page can be executed. -#define PROT_EXEC 0x1 -// Page cannot be accessed. -#define PROT_NONE 0x2 -// Page can be read. -#define PROT_READ 0x4 -// Page can be written. -#define PROT_WRITE 0x8 - - -// Interpret addr exactly. -#define MAP_FIXED 0x1 -// Changes are private. -#define MAP_PRIVATE 0x2 -// Share changes. -#define MAP_SHARED 0x4 - -// Returned on failure to allocate pages -#define MAP_FAILED ((void*)-1) - -#define MREMAP_FIXED 0x2 -#define MREMAP_MAYMOVE 0x4 - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/api/posix/sys/socket.h b/api/posix/sys/socket.h deleted file mode 100644 index 9023b45ba5..0000000000 --- a/api/posix/sys/socket.h +++ /dev/null @@ -1,156 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef POSIX_SYS_SOCKET_H -#define POSIX_SYS_SOCKET_H - -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef uint32_t socklen_t; -typedef uint8_t sa_family_t; - -struct sockaddr -{ - sa_family_t sa_family; // Address family. - char sa_data[]; // Socket address (variable-length data). -}; - -struct sockaddr_storage -{ - sa_family_t sa_family; // Address family. - char sa_data[16] __attribute__((aligned(16))); -}; - -struct msghdr -{ - void *msg_name; // Optional address. - socklen_t msg_namelen; // Size of address. - struct iovec *msg_iov; // Scatter/gather array. - int msg_iovlen; // Members in msg_iov. - void *msg_control; // Ancillary data; see below. - socklen_t msg_controllen; // Ancillary data buffer len. - int msg_flags; // Flags on received message. -}; - -struct cmsghdr -{ - socklen_t cmsg_len; // Data byte count, including the cmsghdr. - int cmsg_level; // Originating protocol. - int cmsg_type; // Protocol-specific type. -}; - -struct linger -{ - int l_onoff; // Indicates whether linger option is enabled. - int l_linger; // Linger time, in seconds. -}; - -#define SOCK_STREAM 0 -#define SOCK_DGRAM 1 -#define SOCK_RAW 2 -#define SOCK_SEQPACKET 3 - -#define SOL_SOCKET 322 - -#define SO_ACCEPTCONN 16 -#define SO_BROADCAST 17 -#define SO_DEBUG 18 -#define SO_DONTROUTE 19 -#define SO_ERROR 20 -#define SO_KEEPALIVE 21 -#define SO_LINGER 22 -#define SO_OOBINLINE 23 -#define SO_RCVBUF 24 -#define SO_RCVLOWAT 25 -#define SO_RCVTIMEO 26 -#define SO_REUSEADDR 27 -#define SO_SNDBUF 28 -#define SO_SNDLOWAT 29 -#define SO_TYPE 30 - -#define SOMAXCONN 64 - -#define AF_UNIX 0 -#define AF_LOCAL AF_UNIX -#define AF_INET 1 -#define AF_INET6 2 -#define AF_UNSPEC 3 - -#define MSG_CTRUNC 0 -#define MSG_DONTROUTE 1 -#define MSG_EOR 2 -#define MSG_OOB 3 -#define MSG_NOSIGNAL 4 -#define MSG_PEEK 5 -#define MSG_TRUNC 6 -#define MSG_WAITALL 7 - - -int accept(int socket, struct sockaddr *address, - socklen_t *address_len); -int bind(int socket, const struct sockaddr *address, - socklen_t address_len); -int connect(int socket, const struct sockaddr *address, - socklen_t address_len); -int getpeername(int socket, struct sockaddr *address, - socklen_t *address_len); -int getsockname(int socket, struct sockaddr *address, - socklen_t *address_len); -int getsockopt(int socket, int level, int option_name, - void *option_value, socklen_t *option_len); -int listen(int socket, int backlog); -ssize_t recv(int socket, void *buffer, size_t length, int flags); -ssize_t recvfrom(int socket, void *buffer, size_t length, - int flags, struct sockaddr *address, socklen_t *address_len); -ssize_t recvmsg(int socket, struct msghdr *message, int flags); -ssize_t send(int socket, const void *message, size_t length, int flags); -ssize_t sendmsg(int socket, const struct msghdr *message, int flags); -ssize_t sendto(int socket, const void *message, size_t length, int flags, - const struct sockaddr *dest_addr, socklen_t dest_len); -int setsockopt(int socket, int level, int option_name, - const void *option_value, socklen_t option_len); -int shutdown(int socket, int how); -int socket(int domain, int type, int protocol); -int socketpair(int domain, int type, int protocol, - int socket_vector[2]); - -enum -{ - SHUT_RD = 0, - SHUT_WR, - SHUT_RDWR -}; - -#define INVALID_SOCKET ((SOCKET)(~0)) -#define SOCKET_ERROR (-1) - -#define SD_RECEIVE SHUT_RD -#define SD_SEND SHUT_WR -#define SD_BOTH SHUT_RDWR - -#include "../netinet/in.h" - -#ifdef __cplusplus -} -#endif -#endif diff --git a/api/posix/sys/uio.h b/api/posix/sys/uio.h deleted file mode 100644 index 7dd8e4948d..0000000000 --- a/api/posix/sys/uio.h +++ /dev/null @@ -1,38 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef POSIX_SYS_UIO_H -#define POSIX_SYS_UIO_H - -#ifdef __cplusplus -extern "C" { -#endif - -struct iovec { - void *iov_base; // Base address of a memory region for input or output - size_t iov_len; // Size of memory pointed to by iov_base -}; - -ssize_t readv(int, const struct iovec *, int); -ssize_t writev(int, const struct iovec *, int); - -#ifdef __cplusplus -} -#endif - -#endif // POSIX_SYS_UIO_H diff --git a/api/posix/sys/utsname.h b/api/posix/sys/utsname.h deleted file mode 100644 index 0d64e5a51b..0000000000 --- a/api/posix/sys/utsname.h +++ /dev/null @@ -1,52 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef POSIX_SYS_UTSNAME_H -#define POSIX_SYS_UTSNAME_H - -#ifdef __cplusplus -extern "C" { -#endif - -static const int LENGTH = 256; - -struct utsname { - /* Name of this implementation of the operating system */ - char sysname[LENGTH]; - - /* Name of this node within the communications network to - which this node is attached, if any */ - char nodename[LENGTH]; - - /* Current release level of this implementation */ - char release[LENGTH]; - - /* Current version level of this release */ - char version[LENGTH]; - - /* Name of the hardware type on which the system is running */ - char machine[LENGTH]; -}; - -int uname(struct utsname *name); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/api/posix/syslog.h b/api/posix/syslog.h deleted file mode 100644 index c15e8d8664..0000000000 --- a/api/posix/syslog.h +++ /dev/null @@ -1,155 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef POSIX_SYSLOG_H -#define POSIX_SYSLOG_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define LOG_PRIMASK 0x07 /* mask to extract priority part (internal) */ - -#define LOG_PRI(p) ((p) & LOG_PRIMASK) - -#define LOG_FACMASK 0x03f8 /* facility mask: mask to extract facility part */ - -/* - Macros for constructing the maskpri argument to setlogmask() - The macros expand to an expression of type int when the argument - pri is an expression of type int -*/ -#define LOG_MASK(pri) (1 << (pri)) /* mask for one priority */ -// Added: Mask of all priorities up to and including pri: -#define LOG_UPTO(pri) ((1 << ((pri)+1)) - 1) /* all priorities through pri */ - -/* - Possible facility values - Symbolic constants for use as the facility argument to openlog() - As specified in RFC5424 -*/ -#define LOG_KERN 0 /* Kernel messages */ -#define LOG_USER 1 /* User-level messages */ -#define LOG_MAIL 2 /* Mail system */ -#define LOG_DAEMON 3 /* System daemons */ -#define LOG_AUTH 4 /* Security/authorization messages */ -#define LOG_LPR 6 /* Line printer subsystem */ -#define LOG_NEWS 7 /* Network news subsystem */ -#define LOG_UUCP 8 /* UUCP subsystem */ -#define LOG_CRON 9 /* Clock daemon */ - -#define LOG_LOCAL0 16 /* Local use 0 */ -#define LOG_LOCAL1 17 /* Local use 1 */ -#define LOG_LOCAL2 18 /* Local use 2 */ -#define LOG_LOCAL3 19 /* Local use 3 */ -#define LOG_LOCAL4 20 /* Local use 4 */ -#define LOG_LOCAL5 21 /* Local use 5 */ -#define LOG_LOCAL6 22 /* Local use 6 */ -#define LOG_LOCAL7 23 /* Local use 7 */ - -/* - Symbolic constants, where zero or more of which may be OR'ed together - to form the logopt option of openlog() -*/ -#define LOG_PID 0X01 /* Log the process ID with each message */ -#define LOG_CONS 0X02 /* Log to the system console on error */ -#define LOG_NDELAY 0X04 /* Connect to syslog daemon immediately */ -#define LOG_ODELAY 0X08 /* Delay open until syslog() is called - Is the converse of LOG_NDELAY and does no longer do anything */ -#define LOG_NOWAIT 0X10 /* Do not wait for child processes - Deprecated */ -#define LOG_PERROR 0X20 /* Is not specified by POSIX.1-2001 or POSIX.1-2008, but is - available in most versions of UNIX - Log to stderr */ - -/* - Possible values of severity - Symbolic constants for use as the priority argument of syslog() - As specified in RFC5424 and POSIX -*/ -#define LOG_EMERG 0 /* Emergency: System is unusable */ -#define LOG_ALERT 1 /* Alert: Action must be taken immediately */ -#define LOG_CRIT 2 /* Critical: Critical conditions */ -#define LOG_ERR 3 /* Error: Error conditions */ -#define LOG_WARNING 4 /* Warning: Warning conditions */ -#define LOG_NOTICE 5 /* Notice: Normal but significant condition */ -#define LOG_INFO 6 /* Informational: Informational messages */ -#define LOG_DEBUG 7 /* Debug: Debug-level messages */ - -/* - Shall close any open file descriptors allocated by previous calls to - openlog() or syslog() -*/ -void closelog(void); - -/* - Shall set process attributes that affect subsequent calls to syslog(). - The ident argument is a string that is prepended to every message. - The logopt argument indicates logging options. - Values for logopt are constructed by a bitwise-inclusive OR of zero or - more of the following: look above. - - The facility argument encodes a default facility to be assigned to all - messages that do not have an explicit facility already encoded. The initial - default facility is LOG_USER. -*/ -void openlog(const char* ident, int logopt, int facility); - -/* - Shall set the log priority mask for the current process to maskpri and - return the previous mask. - If the maskpri argument is 0, the current log mask is not modified. - Calls by the current process to syslog() with a priority not set in maskpri - shall be rejected. - The default log mask allows all priorities to be logged. - A call to openlog() is not required prior to calling setlogmask(). - - The function shall return the previous log priority mask. -*/ -int setlogmask(int maskpri); - -/* - Shall send a message to an implementation-defined logging facility, - which may log it in an implementation-defined system log, write it to the - system console, forward it to a list of users, or forward it to the logging - facility on another host over the network. The logged message shall include - a message header and a message body. The message header contains at least a - timestamp and a tag string. - - The message body is generated from the message (argument) and following arguments - in the same manner as if these were arguments to printf(), except that the additional - conversion specification %m shall be recognized; it shall convert no arguments, - shall cause the output of the error message string associated with the value of - errno on entry to syslog(), and may be mixed with argument specifications of the - "%n$" form. If a complete conversion specification with the m conversion specifier - character is not just %m, the behavior is undefined. A trailing may be - added if needed. - - Values of the priority argument are formed by OR'ing together a severity-level value - and an optional facility value. If no facility value is specified, the current - default facility value is used. - Possible values of severity: look above. - Possible facility values: look above. -*/ -void syslog(int priority, const char* message, ... /* arguments*/); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/api/posix/tar.h b/api/posix/tar.h deleted file mode 100644 index 617cc47a8a..0000000000 --- a/api/posix/tar.h +++ /dev/null @@ -1,94 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef POSIX_TAR_H -#define POSIX_TAR_H - -#ifdef __cplusplus -extern "C" { -#endif - -#define TMAGIC "ustar" -#define TMAGLEN 6 -#define TVERSION "00" -#define TVERSLEN 2 - -#define REGTYPE '0' -#define AREGTYPE '\0' -#define LNKTYPE '1' -#define SYMTYPE '2' -#define CHRTYPE '3' -#define BLKTYPE '4' -#define DIRTYPE '5' -#define FIFOTYPE '6' -#define CONTTYPE '7' - -#define TSUID 04000 -#define TSGID 02000 -#define TSVTX 01000 -#define TUREAD 00400 -#define TUWRITE 00200 -#define TUEXEC 00100 -#define TGREAD 00040 -#define TGWRITE 00020 -#define TGEXEC 00010 -#define TOREAD 00004 -#define TOWRITE 00002 -#define TOEXEC 00001 - -#define LENGTH_NAME 100 -#define LENGTH_MODE 8 -#define LENGTH_UID 8 -#define LENGTH_GID 8 -#define LENGTH_SIZE 12 -#define LENGTH_MTIME 12 -#define LENGTH_CHECKSUM 8 -#define LENGTH_TYPEFLAG 1 -#define LENGTH_LINKNAME 100 -#define LENGTH_MAGIC 6 -#define LENGTH_VERSION 2 -#define LENGTH_UNAME 32 -#define LENGTH_GNAME 32 -#define LENGTH_DEVMAJOR 8 -#define LENGTH_DEVMINOR 8 -#define LENGTH_PREFIX 155 -#define LENGTH_PAD 12 - -struct Tar_header { - char name[LENGTH_NAME]; // Name of header file entry - char mode[LENGTH_MODE]; // Permission and mode bits - char uid[LENGTH_UID]; // User ID of owner - char gid[LENGTH_GID]; // Group ID of owner - char size[LENGTH_SIZE]; // File size in bytes (octal base) - char mod_time[LENGTH_MTIME]; // Last modification time in numeric Unix time format (octal) - char checksum[LENGTH_CHECKSUM]; // Checksum for header record (6 digit octal number with leading zeroes) - char typeflag; // Type of header entry - char linkname[LENGTH_LINKNAME]; // Target name of link - char magic[LENGTH_MAGIC]; // Ustar indicator - char version[LENGTH_VERSION]; // Ustar version - char uname[LENGTH_UNAME]; // User name of owner - char gname[LENGTH_GNAME]; // Group name of owner - char devmajor[LENGTH_DEVMAJOR]; // Major number of character or block device - char devminor[LENGTH_DEVMINOR]; // Minor number of character or block device - char prefix[LENGTH_PREFIX]; // Prefix for file name - char pad[LENGTH_PAD]; -}; - -#ifdef __cplusplus -} -#endif -#endif diff --git a/api/posix/time.h b/api/posix/time.h deleted file mode 100644 index 4447ce6509..0000000000 --- a/api/posix/time.h +++ /dev/null @@ -1,25 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef TIME_H -#define TIME_H - -// We need our features.h to be included before newlib time.h -#include -#include_next - -#endif diff --git a/api/posix/ucontext.h b/api/posix/ucontext.h deleted file mode 100644 index 33707c917b..0000000000 --- a/api/posix/ucontext.h +++ /dev/null @@ -1,62 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef POSIX_UCONTEXT_H -#define POSIX_UCONTEXT_H - -#include -#include - -struct mcontext_t { - size_t edi; - size_t esi; - size_t ebp; - size_t ebx; - size_t edx; - size_t ecx; - size_t eax; - - uint8_t floating_point_env[28]; - size_t eip; - size_t flags; - size_t ret_esp; -}; - -struct ucontext_t { - ucontext_t* uc_link; - stack_t uc_stack; - mcontext_t uc_mcontext; - -public: - static constexpr size_t MAX_STACK_SIZE = 4096 * 3; // Bytes -}; - -#ifdef __cplusplus -extern "C" { -#endif - -int getcontext(ucontext_t *ucp); -int setcontext(const ucontext_t *ucp); -void makecontext(ucontext_t *ucp, void (*func)(), int argc, ...); -int swapcontext(ucontext_t *oucp, ucontext_t *ucp); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/api/posix/unistd.h b/api/posix/unistd.h deleted file mode 100644 index 2357b79b13..0000000000 --- a/api/posix/unistd.h +++ /dev/null @@ -1,35 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#ifndef POSIX_UNISTD_H -#define POSIX_UNISTD_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include_next - -int ftruncate(int fd, off_t length); -int lstat(const char *path, struct stat *buf); - -#ifdef __cplusplus -} -#endif -#endif diff --git a/api/posix/wchar.h b/api/posix/wchar.h deleted file mode 100644 index 0e3b142168..0000000000 --- a/api/posix/wchar.h +++ /dev/null @@ -1,36 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include_next - -#ifndef SYS_WCHAR_H -#define SYS_WCHAR_H - -#ifdef __cplusplus -extern "C" { -#endif - -/** @WARNING Loses precision **/ -inline long double wcstold (const wchar_t* str, wchar_t** endptr) { - return wcstod(str,endptr); -} - -#ifdef __cplusplus -} -#endif - -#endif // SYS_WCHAR_H diff --git a/api/util/elf.h b/api/util/elf.h deleted file mode 100644 index 4f171d47bd..0000000000 --- a/api/util/elf.h +++ /dev/null @@ -1,2813 +0,0 @@ -/* This is the original elf.h file from the GNU C Library; I only removed - the inclusion of feature.h, which is not needed. - - On OSX, simply copy the file to /usr/local/include/. - - Mathias Lafeldt */ - -/* This file defines standard ELF types, structures, and macros. - Copyright (C) 1995-2003,2004,2005,2006,2007,2008,2009,2010,2011 - Free Software Foundation, Inc. - This file is part of the GNU C Library. - - The GNU C Library is free software; you can redistribute it and/or - modify it under the terms of the GNU Lesser General Public - License as published by the Free Software Foundation; either - version 2.1 of the License, or (at your option) any later version. - - The GNU C Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public - License along with the GNU C Library; if not, write to the Free - Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA - 02111-1307 USA. */ - -#ifndef _ELF_H -#define _ELF_H 1 - -#include - -/* C++ needs to know that types and declarations are C, not C++. */ -#ifdef __cplusplus -# define __BEGIN_DECLS extern "C" { -# define __END_DECLS } -#else -# define __BEGIN_DECLS -# define __END_DECLS -#endif - - -__BEGIN_DECLS - -/* Standard ELF types. */ - -#include - -/* Type for a 16-bit quantity. */ -typedef uint16_t Elf32_Half; -typedef uint16_t Elf64_Half; - -/* Types for signed and unsigned 32-bit quantities. */ -typedef uint32_t Elf32_Word; -typedef int32_t Elf32_Sword; -typedef uint32_t Elf64_Word; -typedef int32_t Elf64_Sword; - -/* Types for signed and unsigned 64-bit quantities. */ -typedef uint64_t Elf32_Xword; -typedef int64_t Elf32_Sxword; -typedef uint64_t Elf64_Xword; -typedef int64_t Elf64_Sxword; - -/* Type of addresses. */ -typedef uint32_t Elf32_Addr; -typedef uint64_t Elf64_Addr; - -/* Type of file offsets. */ -typedef uint32_t Elf32_Off; -typedef uint64_t Elf64_Off; - -/* Type for section indices, which are 16-bit quantities. */ -typedef uint16_t Elf32_Section; -typedef uint16_t Elf64_Section; - -/* Type for version symbol information. */ -typedef Elf32_Half Elf32_Versym; -typedef Elf64_Half Elf64_Versym; - - -/* The ELF file header. This appears at the start of every ELF file. */ - -#define EI_NIDENT (16) - -typedef struct -{ - unsigned char e_ident[EI_NIDENT]; /* Magic number and other info */ - Elf32_Half e_type; /* Object file type */ - Elf32_Half e_machine; /* Architecture */ - Elf32_Word e_version; /* Object file version */ - Elf32_Addr e_entry; /* Entry point virtual address */ - Elf32_Off e_phoff; /* Program header table file offset */ - Elf32_Off e_shoff; /* Section header table file offset */ - Elf32_Word e_flags; /* Processor-specific flags */ - Elf32_Half e_ehsize; /* ELF header size in bytes */ - Elf32_Half e_phentsize; /* Program header table entry size */ - Elf32_Half e_phnum; /* Program header table entry count */ - Elf32_Half e_shentsize; /* Section header table entry size */ - Elf32_Half e_shnum; /* Section header table entry count */ - Elf32_Half e_shstrndx; /* Section header string table index */ -} Elf32_Ehdr; - -typedef struct -{ - unsigned char e_ident[EI_NIDENT]; /* Magic number and other info */ - Elf64_Half e_type; /* Object file type */ - Elf64_Half e_machine; /* Architecture */ - Elf64_Word e_version; /* Object file version */ - Elf64_Addr e_entry; /* Entry point virtual address */ - Elf64_Off e_phoff; /* Program header table file offset */ - Elf64_Off e_shoff; /* Section header table file offset */ - Elf64_Word e_flags; /* Processor-specific flags */ - Elf64_Half e_ehsize; /* ELF header size in bytes */ - Elf64_Half e_phentsize; /* Program header table entry size */ - Elf64_Half e_phnum; /* Program header table entry count */ - Elf64_Half e_shentsize; /* Section header table entry size */ - Elf64_Half e_shnum; /* Section header table entry count */ - Elf64_Half e_shstrndx; /* Section header string table index */ -} Elf64_Ehdr; - -/* Fields in the e_ident array. The EI_* macros are indices into the - array. The macros under each EI_* macro are the values the byte - may have. */ - -#define EI_MAG0 0 /* File identification byte 0 index */ -#define ELFMAG0 0x7f /* Magic number byte 0 */ - -#define EI_MAG1 1 /* File identification byte 1 index */ -#define ELFMAG1 'E' /* Magic number byte 1 */ - -#define EI_MAG2 2 /* File identification byte 2 index */ -#define ELFMAG2 'L' /* Magic number byte 2 */ - -#define EI_MAG3 3 /* File identification byte 3 index */ -#define ELFMAG3 'F' /* Magic number byte 3 */ - -/* Conglomeration of the identification bytes, for easy testing as a word. */ -#define ELFMAG "\177ELF" -#define SELFMAG 4 - -#define EI_CLASS 4 /* File class byte index */ -#define ELFCLASSNONE 0 /* Invalid class */ -#define ELFCLASS32 1 /* 32-bit objects */ -#define ELFCLASS64 2 /* 64-bit objects */ -#define ELFCLASSNUM 3 - -#define EI_DATA 5 /* Data encoding byte index */ -#define ELFDATANONE 0 /* Invalid data encoding */ -#define ELFDATA2LSB 1 /* 2's complement, little endian */ -#define ELFDATA2MSB 2 /* 2's complement, big endian */ -#define ELFDATANUM 3 - -#define EI_VERSION 6 /* File version byte index */ - /* Value must be EV_CURRENT */ - -#define EI_OSABI 7 /* OS ABI identification */ -#define ELFOSABI_NONE 0 /* UNIX System V ABI */ -#define ELFOSABI_SYSV 0 /* Alias. */ -#define ELFOSABI_HPUX 1 /* HP-UX */ -#define ELFOSABI_NETBSD 2 /* NetBSD. */ -#define ELFOSABI_GNU 3 /* Object uses GNU ELF extensions. */ -#define ELFOSABI_LINUX ELFOSABI_GNU /* Compatibility alias. */ -#define ELFOSABI_SOLARIS 6 /* Sun Solaris. */ -#define ELFOSABI_AIX 7 /* IBM AIX. */ -#define ELFOSABI_IRIX 8 /* SGI Irix. */ -#define ELFOSABI_FREEBSD 9 /* FreeBSD. */ -#define ELFOSABI_TRU64 10 /* Compaq TRU64 UNIX. */ -#define ELFOSABI_MODESTO 11 /* Novell Modesto. */ -#define ELFOSABI_OPENBSD 12 /* OpenBSD. */ -#define ELFOSABI_ARM_AEABI 64 /* ARM EABI */ -#define ELFOSABI_ARM 97 /* ARM */ -#define ELFOSABI_STANDALONE 255 /* Standalone (embedded) application */ - -#define EI_ABIVERSION 8 /* ABI version */ - -#define EI_PAD 9 /* Byte index of padding bytes */ - -/* Legal values for e_type (object file type). */ - -#define ET_NONE 0 /* No file type */ -#define ET_REL 1 /* Relocatable file */ -#define ET_EXEC 2 /* Executable file */ -#define ET_DYN 3 /* Shared object file */ -#define ET_CORE 4 /* Core file */ -#define ET_NUM 5 /* Number of defined types */ -#define ET_LOOS 0xfe00 /* OS-specific range start */ -#define ET_HIOS 0xfeff /* OS-specific range end */ -#define ET_LOPROC 0xff00 /* Processor-specific range start */ -#define ET_HIPROC 0xffff /* Processor-specific range end */ - -/* Legal values for e_machine (architecture). */ - -#define EM_NONE 0 /* No machine */ -#define EM_M32 1 /* AT&T WE 32100 */ -#define EM_SPARC 2 /* SUN SPARC */ -#define EM_386 3 /* Intel 80386 */ -#define EM_68K 4 /* Motorola m68k family */ -#define EM_88K 5 /* Motorola m88k family */ -#define EM_860 7 /* Intel 80860 */ -#define EM_MIPS 8 /* MIPS R3000 big-endian */ -#define EM_S370 9 /* IBM System/370 */ -#define EM_MIPS_RS3_LE 10 /* MIPS R3000 little-endian */ - -#define EM_PARISC 15 /* HPPA */ -#define EM_VPP500 17 /* Fujitsu VPP500 */ -#define EM_SPARC32PLUS 18 /* Sun's "v8plus" */ -#define EM_960 19 /* Intel 80960 */ -#define EM_PPC 20 /* PowerPC */ -#define EM_PPC64 21 /* PowerPC 64-bit */ -#define EM_S390 22 /* IBM S390 */ - -#define EM_V800 36 /* NEC V800 series */ -#define EM_FR20 37 /* Fujitsu FR20 */ -#define EM_RH32 38 /* TRW RH-32 */ -#define EM_RCE 39 /* Motorola RCE */ -#define EM_ARM 40 /* ARM */ -#define EM_FAKE_ALPHA 41 /* Digital Alpha */ -#define EM_SH 42 /* Hitachi SH */ -#define EM_SPARCV9 43 /* SPARC v9 64-bit */ -#define EM_TRICORE 44 /* Siemens Tricore */ -#define EM_ARC 45 /* Argonaut RISC Core */ -#define EM_H8_300 46 /* Hitachi H8/300 */ -#define EM_H8_300H 47 /* Hitachi H8/300H */ -#define EM_H8S 48 /* Hitachi H8S */ -#define EM_H8_500 49 /* Hitachi H8/500 */ -#define EM_IA_64 50 /* Intel Merced */ -#define EM_MIPS_X 51 /* Stanford MIPS-X */ -#define EM_COLDFIRE 52 /* Motorola Coldfire */ -#define EM_68HC12 53 /* Motorola M68HC12 */ -#define EM_MMA 54 /* Fujitsu MMA Multimedia Accelerator*/ -#define EM_PCP 55 /* Siemens PCP */ -#define EM_NCPU 56 /* Sony nCPU embeeded RISC */ -#define EM_NDR1 57 /* Denso NDR1 microprocessor */ -#define EM_STARCORE 58 /* Motorola Start*Core processor */ -#define EM_ME16 59 /* Toyota ME16 processor */ -#define EM_ST100 60 /* STMicroelectronic ST100 processor */ -#define EM_TINYJ 61 /* Advanced Logic Corp. Tinyj emb.fam*/ -#define EM_X86_64 62 /* AMD x86-64 architecture */ -#define EM_PDSP 63 /* Sony DSP Processor */ - -#define EM_FX66 66 /* Siemens FX66 microcontroller */ -#define EM_ST9PLUS 67 /* STMicroelectronics ST9+ 8/16 mc */ -#define EM_ST7 68 /* STmicroelectronics ST7 8 bit mc */ -#define EM_68HC16 69 /* Motorola MC68HC16 microcontroller */ -#define EM_68HC11 70 /* Motorola MC68HC11 microcontroller */ -#define EM_68HC08 71 /* Motorola MC68HC08 microcontroller */ -#define EM_68HC05 72 /* Motorola MC68HC05 microcontroller */ -#define EM_SVX 73 /* Silicon Graphics SVx */ -#define EM_ST19 74 /* STMicroelectronics ST19 8 bit mc */ -#define EM_VAX 75 /* Digital VAX */ -#define EM_CRIS 76 /* Axis Communications 32-bit embedded processor */ -#define EM_JAVELIN 77 /* Infineon Technologies 32-bit embedded processor */ -#define EM_FIREPATH 78 /* Element 14 64-bit DSP Processor */ -#define EM_ZSP 79 /* LSI Logic 16-bit DSP Processor */ -#define EM_MMIX 80 /* Donald Knuth's educational 64-bit processor */ -#define EM_HUANY 81 /* Harvard University machine-independent object files */ -#define EM_PRISM 82 /* SiTera Prism */ -#define EM_AVR 83 /* Atmel AVR 8-bit microcontroller */ -#define EM_FR30 84 /* Fujitsu FR30 */ -#define EM_D10V 85 /* Mitsubishi D10V */ -#define EM_D30V 86 /* Mitsubishi D30V */ -#define EM_V850 87 /* NEC v850 */ -#define EM_M32R 88 /* Mitsubishi M32R */ -#define EM_MN10300 89 /* Matsushita MN10300 */ -#define EM_MN10200 90 /* Matsushita MN10200 */ -#define EM_PJ 91 /* picoJava */ -#define EM_OPENRISC 92 /* OpenRISC 32-bit embedded processor */ -#define EM_ARC_A5 93 /* ARC Cores Tangent-A5 */ -#define EM_XTENSA 94 /* Tensilica Xtensa Architecture */ -#define EM_NUM 95 - -/* If it is necessary to assign new unofficial EM_* values, please - pick large random numbers (0x8523, 0xa7f2, etc.) to minimize the - chances of collision with official or non-GNU unofficial values. */ - -#define EM_ALPHA 0x9026 - -/* Legal values for e_version (version). */ - -#define EV_NONE 0 /* Invalid ELF version */ -#define EV_CURRENT 1 /* Current version */ -#define EV_NUM 2 - -/* Section header. */ - -typedef struct -{ - Elf32_Word sh_name; /* Section name (string tbl index) */ - Elf32_Word sh_type; /* Section type */ - Elf32_Word sh_flags; /* Section flags */ - Elf32_Addr sh_addr; /* Section virtual addr at execution */ - Elf32_Off sh_offset; /* Section file offset */ - Elf32_Word sh_size; /* Section size in bytes */ - Elf32_Word sh_link; /* Link to another section */ - Elf32_Word sh_info; /* Additional section information */ - Elf32_Word sh_addralign; /* Section alignment */ - Elf32_Word sh_entsize; /* Entry size if section holds table */ -} Elf32_Shdr; - -typedef struct -{ - Elf64_Word sh_name; /* Section name (string tbl index) */ - Elf64_Word sh_type; /* Section type */ - Elf64_Xword sh_flags; /* Section flags */ - Elf64_Addr sh_addr; /* Section virtual addr at execution */ - Elf64_Off sh_offset; /* Section file offset */ - Elf64_Xword sh_size; /* Section size in bytes */ - Elf64_Word sh_link; /* Link to another section */ - Elf64_Word sh_info; /* Additional section information */ - Elf64_Xword sh_addralign; /* Section alignment */ - Elf64_Xword sh_entsize; /* Entry size if section holds table */ -} Elf64_Shdr; - -/* Special section indices. */ - -#define SHN_UNDEF 0 /* Undefined section */ -#define SHN_LORESERVE 0xff00 /* Start of reserved indices */ -#define SHN_LOPROC 0xff00 /* Start of processor-specific */ -#define SHN_BEFORE 0xff00 /* Order section before all others - (Solaris). */ -#define SHN_AFTER 0xff01 /* Order section after all others - (Solaris). */ -#define SHN_HIPROC 0xff1f /* End of processor-specific */ -#define SHN_LOOS 0xff20 /* Start of OS-specific */ -#define SHN_HIOS 0xff3f /* End of OS-specific */ -#define SHN_ABS 0xfff1 /* Associated symbol is absolute */ -#define SHN_COMMON 0xfff2 /* Associated symbol is common */ -#define SHN_XINDEX 0xffff /* Index is in extra table. */ -#define SHN_HIRESERVE 0xffff /* End of reserved indices */ - -/* Legal values for sh_type (section type). */ - -#define SHT_NULL 0 /* Section header table entry unused */ -#define SHT_PROGBITS 1 /* Program data */ -#define SHT_SYMTAB 2 /* Symbol table */ -#define SHT_STRTAB 3 /* String table */ -#define SHT_RELA 4 /* Relocation entries with addends */ -#define SHT_HASH 5 /* Symbol hash table */ -#define SHT_DYNAMIC 6 /* Dynamic linking information */ -#define SHT_NOTE 7 /* Notes */ -#define SHT_NOBITS 8 /* Program space with no data (bss) */ -#define SHT_REL 9 /* Relocation entries, no addends */ -#define SHT_SHLIB 10 /* Reserved */ -#define SHT_DYNSYM 11 /* Dynamic linker symbol table */ -#define SHT_INIT_ARRAY 14 /* Array of constructors */ -#define SHT_FINI_ARRAY 15 /* Array of destructors */ -#define SHT_PREINIT_ARRAY 16 /* Array of pre-constructors */ -#define SHT_GROUP 17 /* Section group */ -#define SHT_SYMTAB_SHNDX 18 /* Extended section indeces */ -#define SHT_NUM 19 /* Number of defined types. */ -#define SHT_LOOS 0x60000000 /* Start OS-specific. */ -#define SHT_GNU_ATTRIBUTES 0x6ffffff5 /* Object attributes. */ -#define SHT_GNU_HASH 0x6ffffff6 /* GNU-style hash table. */ -#define SHT_GNU_LIBLIST 0x6ffffff7 /* Prelink library list */ -#define SHT_CHECKSUM 0x6ffffff8 /* Checksum for DSO content. */ -#define SHT_LOSUNW 0x6ffffffa /* Sun-specific low bound. */ -#define SHT_SUNW_move 0x6ffffffa -#define SHT_SUNW_COMDAT 0x6ffffffb -#define SHT_SUNW_syminfo 0x6ffffffc -#define SHT_GNU_verdef 0x6ffffffd /* Version definition section. */ -#define SHT_GNU_verneed 0x6ffffffe /* Version needs section. */ -#define SHT_GNU_versym 0x6fffffff /* Version symbol table. */ -#define SHT_HISUNW 0x6fffffff /* Sun-specific high bound. */ -#define SHT_HIOS 0x6fffffff /* End OS-specific type */ -#define SHT_LOPROC 0x70000000 /* Start of processor-specific */ -#define SHT_HIPROC 0x7fffffff /* End of processor-specific */ -#define SHT_LOUSER 0x80000000 /* Start of application-specific */ -#define SHT_HIUSER 0x8fffffff /* End of application-specific */ - -/* Legal values for sh_flags (section flags). */ - -#define SHF_WRITE (1 << 0) /* Writable */ -#define SHF_ALLOC (1 << 1) /* Occupies memory during execution */ -#define SHF_EXECINSTR (1 << 2) /* Executable */ -#define SHF_MERGE (1 << 4) /* Might be merged */ -#define SHF_STRINGS (1 << 5) /* Contains nul-terminated strings */ -#define SHF_INFO_LINK (1 << 6) /* `sh_info' contains SHT index */ -#define SHF_LINK_ORDER (1 << 7) /* Preserve order after combining */ -#define SHF_OS_NONCONFORMING (1 << 8) /* Non-standard OS specific handling - required */ -#define SHF_GROUP (1 << 9) /* Section is member of a group. */ -#define SHF_TLS (1 << 10) /* Section hold thread-local data. */ -#define SHF_MASKOS 0x0ff00000 /* OS-specific. */ -#define SHF_MASKPROC 0xf0000000 /* Processor-specific */ -#define SHF_ORDERED (1 << 30) /* Special ordering requirement - (Solaris). */ -#define SHF_EXCLUDE (1 << 31) /* Section is excluded unless - referenced or allocated (Solaris).*/ - -/* Section group handling. */ -#define GRP_COMDAT 0x1 /* Mark group as COMDAT. */ - -/* Symbol table entry. */ - -typedef struct -{ - Elf32_Word st_name; /* Symbol name (string tbl index) */ - Elf32_Addr st_value; /* Symbol value */ - Elf32_Word st_size; /* Symbol size */ - unsigned char st_info; /* Symbol type and binding */ - unsigned char st_other; /* Symbol visibility */ - Elf32_Section st_shndx; /* Section index */ -} Elf32_Sym; - -typedef struct -{ - Elf64_Word st_name; /* Symbol name (string tbl index) */ - unsigned char st_info; /* Symbol type and binding */ - unsigned char st_other; /* Symbol visibility */ - Elf64_Section st_shndx; /* Section index */ - Elf64_Addr st_value; /* Symbol value */ - Elf64_Xword st_size; /* Symbol size */ -} Elf64_Sym; - -/* The syminfo section if available contains additional information about - every dynamic symbol. */ - -typedef struct -{ - Elf32_Half si_boundto; /* Direct bindings, symbol bound to */ - Elf32_Half si_flags; /* Per symbol flags */ -} Elf32_Syminfo; - -typedef struct -{ - Elf64_Half si_boundto; /* Direct bindings, symbol bound to */ - Elf64_Half si_flags; /* Per symbol flags */ -} Elf64_Syminfo; - -/* Possible values for si_boundto. */ -#define SYMINFO_BT_SELF 0xffff /* Symbol bound to self */ -#define SYMINFO_BT_PARENT 0xfffe /* Symbol bound to parent */ -#define SYMINFO_BT_LOWRESERVE 0xff00 /* Beginning of reserved entries */ - -/* Possible bitmasks for si_flags. */ -#define SYMINFO_FLG_DIRECT 0x0001 /* Direct bound symbol */ -#define SYMINFO_FLG_PASSTHRU 0x0002 /* Pass-thru symbol for translator */ -#define SYMINFO_FLG_COPY 0x0004 /* Symbol is a copy-reloc */ -#define SYMINFO_FLG_LAZYLOAD 0x0008 /* Symbol bound to object to be lazy - loaded */ -/* Syminfo version values. */ -#define SYMINFO_NONE 0 -#define SYMINFO_CURRENT 1 -#define SYMINFO_NUM 2 - - -/* How to extract and insert information held in the st_info field. */ - -#define ELF32_ST_BIND(val) (((unsigned char) (val)) >> 4) -#define ELF32_ST_TYPE(val) ((val) & 0xf) -#define ELF32_ST_INFO(bind, type) (((bind) << 4) + ((type) & 0xf)) - -/* Both Elf32_Sym and Elf64_Sym use the same one-byte st_info field. */ -#define ELF64_ST_BIND(val) ELF32_ST_BIND (val) -#define ELF64_ST_TYPE(val) ELF32_ST_TYPE (val) -#define ELF64_ST_INFO(bind, type) ELF32_ST_INFO ((bind), (type)) - -/* Legal values for ST_BIND subfield of st_info (symbol binding). */ - -#define STB_LOCAL 0 /* Local symbol */ -#define STB_GLOBAL 1 /* Global symbol */ -#define STB_WEAK 2 /* Weak symbol */ -#define STB_NUM 3 /* Number of defined types. */ -#define STB_LOOS 10 /* Start of OS-specific */ -#define STB_GNU_UNIQUE 10 /* Unique symbol. */ -#define STB_HIOS 12 /* End of OS-specific */ -#define STB_LOPROC 13 /* Start of processor-specific */ -#define STB_HIPROC 15 /* End of processor-specific */ - -/* Legal values for ST_TYPE subfield of st_info (symbol type). */ - -#define STT_NOTYPE 0 /* Symbol type is unspecified */ -#define STT_OBJECT 1 /* Symbol is a data object */ -#define STT_FUNC 2 /* Symbol is a code object */ -#define STT_SECTION 3 /* Symbol associated with a section */ -#define STT_FILE 4 /* Symbol's name is file name */ -#define STT_COMMON 5 /* Symbol is a common data object */ -#define STT_TLS 6 /* Symbol is thread-local data object*/ -#define STT_NUM 7 /* Number of defined types. */ -#define STT_LOOS 10 /* Start of OS-specific */ -#define STT_GNU_IFUNC 10 /* Symbol is indirect code object */ -#define STT_HIOS 12 /* End of OS-specific */ -#define STT_LOPROC 13 /* Start of processor-specific */ -#define STT_HIPROC 15 /* End of processor-specific */ - - -/* Symbol table indices are found in the hash buckets and chain table - of a symbol hash table section. This special index value indicates - the end of a chain, meaning no further symbols are found in that bucket. */ - -#define STN_UNDEF 0 /* End of a chain. */ - - -/* How to extract and insert information held in the st_other field. */ - -#define ELF32_ST_VISIBILITY(o) ((o) & 0x03) - -/* For ELF64 the definitions are the same. */ -#define ELF64_ST_VISIBILITY(o) ELF32_ST_VISIBILITY (o) - -/* Symbol visibility specification encoded in the st_other field. */ -#define STV_DEFAULT 0 /* Default symbol visibility rules */ -#define STV_INTERNAL 1 /* Processor specific hidden class */ -#define STV_HIDDEN 2 /* Sym unavailable in other modules */ -#define STV_PROTECTED 3 /* Not preemptible, not exported */ - - -/* Relocation table entry without addend (in section of type SHT_REL). */ - -typedef struct -{ - Elf32_Addr r_offset; /* Address */ - Elf32_Word r_info; /* Relocation type and symbol index */ -} Elf32_Rel; - -/* I have seen two different definitions of the Elf64_Rel and - Elf64_Rela structures, so we'll leave them out until Novell (or - whoever) gets their act together. */ -/* The following, at least, is used on Sparc v9, MIPS, and Alpha. */ - -typedef struct -{ - Elf64_Addr r_offset; /* Address */ - Elf64_Xword r_info; /* Relocation type and symbol index */ -} Elf64_Rel; - -/* Relocation table entry with addend (in section of type SHT_RELA). */ - -typedef struct -{ - Elf32_Addr r_offset; /* Address */ - Elf32_Word r_info; /* Relocation type and symbol index */ - Elf32_Sword r_addend; /* Addend */ -} Elf32_Rela; - -typedef struct -{ - Elf64_Addr r_offset; /* Address */ - Elf64_Xword r_info; /* Relocation type and symbol index */ - Elf64_Sxword r_addend; /* Addend */ -} Elf64_Rela; - -/* How to extract and insert information held in the r_info field. */ - -#define ELF32_R_SYM(val) ((val) >> 8) -#define ELF32_R_TYPE(val) ((val) & 0xff) -#define ELF32_R_INFO(sym, type) (((sym) << 8) + ((type) & 0xff)) - -#define ELF64_R_SYM(i) ((i) >> 32) -#define ELF64_R_TYPE(i) ((i) & 0xffffffff) -#define ELF64_R_INFO(sym,type) ((((Elf64_Xword) (sym)) << 32) + (type)) - -/* Program segment header. */ - -typedef struct -{ - Elf32_Word p_type; /* Segment type */ - Elf32_Off p_offset; /* Segment file offset */ - Elf32_Addr p_vaddr; /* Segment virtual address */ - Elf32_Addr p_paddr; /* Segment physical address */ - Elf32_Word p_filesz; /* Segment size in file */ - Elf32_Word p_memsz; /* Segment size in memory */ - Elf32_Word p_flags; /* Segment flags */ - Elf32_Word p_align; /* Segment alignment */ -} Elf32_Phdr; - -typedef struct -{ - Elf64_Word p_type; /* Segment type */ - Elf64_Word p_flags; /* Segment flags */ - Elf64_Off p_offset; /* Segment file offset */ - Elf64_Addr p_vaddr; /* Segment virtual address */ - Elf64_Addr p_paddr; /* Segment physical address */ - Elf64_Xword p_filesz; /* Segment size in file */ - Elf64_Xword p_memsz; /* Segment size in memory */ - Elf64_Xword p_align; /* Segment alignment */ -} Elf64_Phdr; - -/* Special value for e_phnum. This indicates that the real number of - program headers is too large to fit into e_phnum. Instead the real - value is in the field sh_info of section 0. */ - -#define PN_XNUM 0xffff - -/* Legal values for p_type (segment type). */ - -#define PT_NULL 0 /* Program header table entry unused */ -#define PT_LOAD 1 /* Loadable program segment */ -#define PT_DYNAMIC 2 /* Dynamic linking information */ -#define PT_INTERP 3 /* Program interpreter */ -#define PT_NOTE 4 /* Auxiliary information */ -#define PT_SHLIB 5 /* Reserved */ -#define PT_PHDR 6 /* Entry for header table itself */ -#define PT_TLS 7 /* Thread-local storage segment */ -#define PT_NUM 8 /* Number of defined types */ -#define PT_LOOS 0x60000000 /* Start of OS-specific */ -#define PT_GNU_EH_FRAME 0x6474e550 /* GCC .eh_frame_hdr segment */ -#define PT_GNU_STACK 0x6474e551 /* Indicates stack executability */ -#define PT_GNU_RELRO 0x6474e552 /* Read-only after relocation */ -#define PT_LOSUNW 0x6ffffffa -#define PT_SUNWBSS 0x6ffffffa /* Sun Specific segment */ -#define PT_SUNWSTACK 0x6ffffffb /* Stack segment */ -#define PT_HISUNW 0x6fffffff -#define PT_HIOS 0x6fffffff /* End of OS-specific */ -#define PT_LOPROC 0x70000000 /* Start of processor-specific */ -#define PT_HIPROC 0x7fffffff /* End of processor-specific */ - -/* Legal values for p_flags (segment flags). */ - -#define PF_X (1 << 0) /* Segment is executable */ -#define PF_W (1 << 1) /* Segment is writable */ -#define PF_R (1 << 2) /* Segment is readable */ -#define PF_MASKOS 0x0ff00000 /* OS-specific */ -#define PF_MASKPROC 0xf0000000 /* Processor-specific */ - -/* Legal values for note segment descriptor types for core files. */ - -#define NT_PRSTATUS 1 /* Contains copy of prstatus struct */ -#define NT_FPREGSET 2 /* Contains copy of fpregset struct */ -#define NT_PRPSINFO 3 /* Contains copy of prpsinfo struct */ -#define NT_PRXREG 4 /* Contains copy of prxregset struct */ -#define NT_TASKSTRUCT 4 /* Contains copy of task structure */ -#define NT_PLATFORM 5 /* String from sysinfo(SI_PLATFORM) */ -#define NT_AUXV 6 /* Contains copy of auxv array */ -#define NT_GWINDOWS 7 /* Contains copy of gwindows struct */ -#define NT_ASRS 8 /* Contains copy of asrset struct */ -#define NT_PSTATUS 10 /* Contains copy of pstatus struct */ -#define NT_PSINFO 13 /* Contains copy of psinfo struct */ -#define NT_PRCRED 14 /* Contains copy of prcred struct */ -#define NT_UTSNAME 15 /* Contains copy of utsname struct */ -#define NT_LWPSTATUS 16 /* Contains copy of lwpstatus struct */ -#define NT_LWPSINFO 17 /* Contains copy of lwpinfo struct */ -#define NT_PRFPXREG 20 /* Contains copy of fprxregset struct */ -#define NT_PRXFPREG 0x46e62b7f /* Contains copy of user_fxsr_struct */ -#define NT_PPC_VMX 0x100 /* PowerPC Altivec/VMX registers */ -#define NT_PPC_SPE 0x101 /* PowerPC SPE/EVR registers */ -#define NT_PPC_VSX 0x102 /* PowerPC VSX registers */ -#define NT_386_TLS 0x200 /* i386 TLS slots (struct user_desc) */ -#define NT_386_IOPERM 0x201 /* x86 io permission bitmap (1=deny) */ -#define NT_X86_XSTATE 0x202 /* x86 extended state using xsave */ - -/* Legal values for the note segment descriptor types for object files. */ - -#define NT_VERSION 1 /* Contains a version string. */ - - -/* Dynamic section entry. */ - -typedef struct -{ - Elf32_Sword d_tag; /* Dynamic entry type */ - union - { - Elf32_Word d_val; /* Integer value */ - Elf32_Addr d_ptr; /* Address value */ - } d_un; -} Elf32_Dyn; - -typedef struct -{ - Elf64_Sxword d_tag; /* Dynamic entry type */ - union - { - Elf64_Xword d_val; /* Integer value */ - Elf64_Addr d_ptr; /* Address value */ - } d_un; -} Elf64_Dyn; - -/* Legal values for d_tag (dynamic entry type). */ - -#define DT_NULL 0 /* Marks end of dynamic section */ -#define DT_NEEDED 1 /* Name of needed library */ -#define DT_PLTRELSZ 2 /* Size in bytes of PLT relocs */ -#define DT_PLTGOT 3 /* Processor defined value */ -#define DT_HASH 4 /* Address of symbol hash table */ -#define DT_STRTAB 5 /* Address of string table */ -#define DT_SYMTAB 6 /* Address of symbol table */ -#define DT_RELA 7 /* Address of Rela relocs */ -#define DT_RELASZ 8 /* Total size of Rela relocs */ -#define DT_RELAENT 9 /* Size of one Rela reloc */ -#define DT_STRSZ 10 /* Size of string table */ -#define DT_SYMENT 11 /* Size of one symbol table entry */ -#define DT_INIT 12 /* Address of init function */ -#define DT_FINI 13 /* Address of termination function */ -#define DT_SONAME 14 /* Name of shared object */ -#define DT_RPATH 15 /* Library search path (deprecated) */ -#define DT_SYMBOLIC 16 /* Start symbol search here */ -#define DT_REL 17 /* Address of Rel relocs */ -#define DT_RELSZ 18 /* Total size of Rel relocs */ -#define DT_RELENT 19 /* Size of one Rel reloc */ -#define DT_PLTREL 20 /* Type of reloc in PLT */ -#define DT_DEBUG 21 /* For debugging; unspecified */ -#define DT_TEXTREL 22 /* Reloc might modify .text */ -#define DT_JMPREL 23 /* Address of PLT relocs */ -#define DT_BIND_NOW 24 /* Process relocations of object */ -#define DT_INIT_ARRAY 25 /* Array with addresses of init fct */ -#define DT_FINI_ARRAY 26 /* Array with addresses of fini fct */ -#define DT_INIT_ARRAYSZ 27 /* Size in bytes of DT_INIT_ARRAY */ -#define DT_FINI_ARRAYSZ 28 /* Size in bytes of DT_FINI_ARRAY */ -#define DT_RUNPATH 29 /* Library search path */ -#define DT_FLAGS 30 /* Flags for the object being loaded */ -#define DT_ENCODING 32 /* Start of encoded range */ -#define DT_PREINIT_ARRAY 32 /* Array with addresses of preinit fct*/ -#define DT_PREINIT_ARRAYSZ 33 /* size in bytes of DT_PREINIT_ARRAY */ -#define DT_NUM 34 /* Number used */ -#define DT_LOOS 0x6000000d /* Start of OS-specific */ -#define DT_HIOS 0x6ffff000 /* End of OS-specific */ -#define DT_LOPROC 0x70000000 /* Start of processor-specific */ -#define DT_HIPROC 0x7fffffff /* End of processor-specific */ -#define DT_PROCNUM DT_MIPS_NUM /* Most used by any processor */ - -/* DT_* entries which fall between DT_VALRNGHI & DT_VALRNGLO use the - Dyn.d_un.d_val field of the Elf*_Dyn structure. This follows Sun's - approach. */ -#define DT_VALRNGLO 0x6ffffd00 -#define DT_GNU_PRELINKED 0x6ffffdf5 /* Prelinking timestamp */ -#define DT_GNU_CONFLICTSZ 0x6ffffdf6 /* Size of conflict section */ -#define DT_GNU_LIBLISTSZ 0x6ffffdf7 /* Size of library list */ -#define DT_CHECKSUM 0x6ffffdf8 -#define DT_PLTPADSZ 0x6ffffdf9 -#define DT_MOVEENT 0x6ffffdfa -#define DT_MOVESZ 0x6ffffdfb -#define DT_FEATURE_1 0x6ffffdfc /* Feature selection (DTF_*). */ -#define DT_POSFLAG_1 0x6ffffdfd /* Flags for DT_* entries, effecting - the following DT_* entry. */ -#define DT_SYMINSZ 0x6ffffdfe /* Size of syminfo table (in bytes) */ -#define DT_SYMINENT 0x6ffffdff /* Entry size of syminfo */ -#define DT_VALRNGHI 0x6ffffdff -#define DT_VALTAGIDX(tag) (DT_VALRNGHI - (tag)) /* Reverse order! */ -#define DT_VALNUM 12 - -/* DT_* entries which fall between DT_ADDRRNGHI & DT_ADDRRNGLO use the - Dyn.d_un.d_ptr field of the Elf*_Dyn structure. - - If any adjustment is made to the ELF object after it has been - built these entries will need to be adjusted. */ -#define DT_ADDRRNGLO 0x6ffffe00 -#define DT_GNU_HASH 0x6ffffef5 /* GNU-style hash table. */ -#define DT_TLSDESC_PLT 0x6ffffef6 -#define DT_TLSDESC_GOT 0x6ffffef7 -#define DT_GNU_CONFLICT 0x6ffffef8 /* Start of conflict section */ -#define DT_GNU_LIBLIST 0x6ffffef9 /* Library list */ -#define DT_CONFIG 0x6ffffefa /* Configuration information. */ -#define DT_DEPAUDIT 0x6ffffefb /* Dependency auditing. */ -#define DT_AUDIT 0x6ffffefc /* Object auditing. */ -#define DT_PLTPAD 0x6ffffefd /* PLT padding. */ -#define DT_MOVETAB 0x6ffffefe /* Move table. */ -#define DT_SYMINFO 0x6ffffeff /* Syminfo table. */ -#define DT_ADDRRNGHI 0x6ffffeff -#define DT_ADDRTAGIDX(tag) (DT_ADDRRNGHI - (tag)) /* Reverse order! */ -#define DT_ADDRNUM 11 - -/* The versioning entry types. The next are defined as part of the - GNU extension. */ -#define DT_VERSYM 0x6ffffff0 - -#define DT_RELACOUNT 0x6ffffff9 -#define DT_RELCOUNT 0x6ffffffa - -/* These were chosen by Sun. */ -#define DT_FLAGS_1 0x6ffffffb /* State flags, see DF_1_* below. */ -#define DT_VERDEF 0x6ffffffc /* Address of version definition - table */ -#define DT_VERDEFNUM 0x6ffffffd /* Number of version definitions */ -#define DT_VERNEED 0x6ffffffe /* Address of table with needed - versions */ -#define DT_VERNEEDNUM 0x6fffffff /* Number of needed versions */ -#define DT_VERSIONTAGIDX(tag) (DT_VERNEEDNUM - (tag)) /* Reverse order! */ -#define DT_VERSIONTAGNUM 16 - -/* Sun added these machine-independent extensions in the "processor-specific" - range. Be compatible. */ -#define DT_AUXILIARY 0x7ffffffd /* Shared object to load before self */ -#define DT_FILTER 0x7fffffff /* Shared object to get values from */ -#define DT_EXTRATAGIDX(tag) ((Elf32_Word)-((Elf32_Sword) (tag) <<1>>1)-1) -#define DT_EXTRANUM 3 - -/* Values of `d_un.d_val' in the DT_FLAGS entry. */ -#define DF_ORIGIN 0x00000001 /* Object may use DF_ORIGIN */ -#define DF_SYMBOLIC 0x00000002 /* Symbol resolutions starts here */ -#define DF_TEXTREL 0x00000004 /* Object contains text relocations */ -#define DF_BIND_NOW 0x00000008 /* No lazy binding for this object */ -#define DF_STATIC_TLS 0x00000010 /* Module uses the static TLS model */ - -/* State flags selectable in the `d_un.d_val' element of the DT_FLAGS_1 - entry in the dynamic section. */ -#define DF_1_NOW 0x00000001 /* Set RTLD_NOW for this object. */ -#define DF_1_GLOBAL 0x00000002 /* Set RTLD_GLOBAL for this object. */ -#define DF_1_GROUP 0x00000004 /* Set RTLD_GROUP for this object. */ -#define DF_1_NODELETE 0x00000008 /* Set RTLD_NODELETE for this object.*/ -#define DF_1_LOADFLTR 0x00000010 /* Trigger filtee loading at runtime.*/ -#define DF_1_INITFIRST 0x00000020 /* Set RTLD_INITFIRST for this object*/ -#define DF_1_NOOPEN 0x00000040 /* Set RTLD_NOOPEN for this object. */ -#define DF_1_ORIGIN 0x00000080 /* $ORIGIN must be handled. */ -#define DF_1_DIRECT 0x00000100 /* Direct binding enabled. */ -#define DF_1_TRANS 0x00000200 -#define DF_1_INTERPOSE 0x00000400 /* Object is used to interpose. */ -#define DF_1_NODEFLIB 0x00000800 /* Ignore default lib search path. */ -#define DF_1_NODUMP 0x00001000 /* Object can't be dldump'ed. */ -#define DF_1_CONFALT 0x00002000 /* Configuration alternative created.*/ -#define DF_1_ENDFILTEE 0x00004000 /* Filtee terminates filters search. */ -#define DF_1_DISPRELDNE 0x00008000 /* Disp reloc applied at build time. */ -#define DF_1_DISPRELPND 0x00010000 /* Disp reloc applied at run-time. */ - -/* Flags for the feature selection in DT_FEATURE_1. */ -#define DTF_1_PARINIT 0x00000001 -#define DTF_1_CONFEXP 0x00000002 - -/* Flags in the DT_POSFLAG_1 entry effecting only the next DT_* entry. */ -#define DF_P1_LAZYLOAD 0x00000001 /* Lazyload following object. */ -#define DF_P1_GROUPPERM 0x00000002 /* Symbols from next object are not - generally available. */ - -/* Version definition sections. */ - -typedef struct -{ - Elf32_Half vd_version; /* Version revision */ - Elf32_Half vd_flags; /* Version information */ - Elf32_Half vd_ndx; /* Version Index */ - Elf32_Half vd_cnt; /* Number of associated aux entries */ - Elf32_Word vd_hash; /* Version name hash value */ - Elf32_Word vd_aux; /* Offset in bytes to verdaux array */ - Elf32_Word vd_next; /* Offset in bytes to next verdef - entry */ -} Elf32_Verdef; - -typedef struct -{ - Elf64_Half vd_version; /* Version revision */ - Elf64_Half vd_flags; /* Version information */ - Elf64_Half vd_ndx; /* Version Index */ - Elf64_Half vd_cnt; /* Number of associated aux entries */ - Elf64_Word vd_hash; /* Version name hash value */ - Elf64_Word vd_aux; /* Offset in bytes to verdaux array */ - Elf64_Word vd_next; /* Offset in bytes to next verdef - entry */ -} Elf64_Verdef; - - -/* Legal values for vd_version (version revision). */ -#define VER_DEF_NONE 0 /* No version */ -#define VER_DEF_CURRENT 1 /* Current version */ -#define VER_DEF_NUM 2 /* Given version number */ - -/* Legal values for vd_flags (version information flags). */ -#define VER_FLG_BASE 0x1 /* Version definition of file itself */ -#define VER_FLG_WEAK 0x2 /* Weak version identifier */ - -/* Versym symbol index values. */ -#define VER_NDX_LOCAL 0 /* Symbol is local. */ -#define VER_NDX_GLOBAL 1 /* Symbol is global. */ -#define VER_NDX_LORESERVE 0xff00 /* Beginning of reserved entries. */ -#define VER_NDX_ELIMINATE 0xff01 /* Symbol is to be eliminated. */ - -/* Auxialiary version information. */ - -typedef struct -{ - Elf32_Word vda_name; /* Version or dependency names */ - Elf32_Word vda_next; /* Offset in bytes to next verdaux - entry */ -} Elf32_Verdaux; - -typedef struct -{ - Elf64_Word vda_name; /* Version or dependency names */ - Elf64_Word vda_next; /* Offset in bytes to next verdaux - entry */ -} Elf64_Verdaux; - - -/* Version dependency section. */ - -typedef struct -{ - Elf32_Half vn_version; /* Version of structure */ - Elf32_Half vn_cnt; /* Number of associated aux entries */ - Elf32_Word vn_file; /* Offset of filename for this - dependency */ - Elf32_Word vn_aux; /* Offset in bytes to vernaux array */ - Elf32_Word vn_next; /* Offset in bytes to next verneed - entry */ -} Elf32_Verneed; - -typedef struct -{ - Elf64_Half vn_version; /* Version of structure */ - Elf64_Half vn_cnt; /* Number of associated aux entries */ - Elf64_Word vn_file; /* Offset of filename for this - dependency */ - Elf64_Word vn_aux; /* Offset in bytes to vernaux array */ - Elf64_Word vn_next; /* Offset in bytes to next verneed - entry */ -} Elf64_Verneed; - - -/* Legal values for vn_version (version revision). */ -#define VER_NEED_NONE 0 /* No version */ -#define VER_NEED_CURRENT 1 /* Current version */ -#define VER_NEED_NUM 2 /* Given version number */ - -/* Auxiliary needed version information. */ - -typedef struct -{ - Elf32_Word vna_hash; /* Hash value of dependency name */ - Elf32_Half vna_flags; /* Dependency specific information */ - Elf32_Half vna_other; /* Unused */ - Elf32_Word vna_name; /* Dependency name string offset */ - Elf32_Word vna_next; /* Offset in bytes to next vernaux - entry */ -} Elf32_Vernaux; - -typedef struct -{ - Elf64_Word vna_hash; /* Hash value of dependency name */ - Elf64_Half vna_flags; /* Dependency specific information */ - Elf64_Half vna_other; /* Unused */ - Elf64_Word vna_name; /* Dependency name string offset */ - Elf64_Word vna_next; /* Offset in bytes to next vernaux - entry */ -} Elf64_Vernaux; - - -/* Legal values for vna_flags. */ -#define VER_FLG_WEAK 0x2 /* Weak version identifier */ - - -/* Auxiliary vector. */ - -/* This vector is normally only used by the program interpreter. The - usual definition in an ABI supplement uses the name auxv_t. The - vector is not usually defined in a standard file, but it - can't hurt. We rename it to avoid conflicts. The sizes of these - types are an arrangement between the exec server and the program - interpreter, so we don't fully specify them here. */ - -typedef struct -{ - uint32_t a_type; /* Entry type */ - union - { - uint32_t a_val; /* Integer value */ - /* We use to have pointer elements added here. We cannot do that, - though, since it does not work when using 32-bit definitions - on 64-bit platforms and vice versa. */ - } a_un; -} Elf32_auxv_t; - -typedef struct -{ - uint64_t a_type; /* Entry type */ - union - { - uint64_t a_val; /* Integer value */ - /* We use to have pointer elements added here. We cannot do that, - though, since it does not work when using 32-bit definitions - on 64-bit platforms and vice versa. */ - } a_un; -} Elf64_auxv_t; - -/* Legal values for a_type (entry type). */ - -#define AT_NULL 0 /* End of vector */ -#define AT_IGNORE 1 /* Entry should be ignored */ -#define AT_EXECFD 2 /* File descriptor of program */ -#define AT_PHDR 3 /* Program headers for program */ -#define AT_PHENT 4 /* Size of program header entry */ -#define AT_PHNUM 5 /* Number of program headers */ -#define AT_PAGESZ 6 /* System page size */ -#define AT_BASE 7 /* Base address of interpreter */ -#define AT_FLAGS 8 /* Flags */ -#define AT_ENTRY 9 /* Entry point of program */ -#define AT_NOTELF 10 /* Program is not ELF */ -#define AT_UID 11 /* Real uid */ -#define AT_EUID 12 /* Effective uid */ -#define AT_GID 13 /* Real gid */ -#define AT_EGID 14 /* Effective gid */ -#define AT_CLKTCK 17 /* Frequency of times() */ - -/* Some more special a_type values describing the hardware. */ -#define AT_PLATFORM 15 /* String identifying platform. */ -#define AT_HWCAP 16 /* Machine dependent hints about - processor capabilities. */ - -/* This entry gives some information about the FPU initialization - performed by the kernel. */ -#define AT_FPUCW 18 /* Used FPU control word. */ - -/* Cache block sizes. */ -#define AT_DCACHEBSIZE 19 /* Data cache block size. */ -#define AT_ICACHEBSIZE 20 /* Instruction cache block size. */ -#define AT_UCACHEBSIZE 21 /* Unified cache block size. */ - -/* A special ignored value for PPC, used by the kernel to control the - interpretation of the AUXV. Must be > 16. */ -#define AT_IGNOREPPC 22 /* Entry should be ignored. */ - -#define AT_SECURE 23 /* Boolean, was exec setuid-like? */ - -#define AT_BASE_PLATFORM 24 /* String identifying real platforms.*/ - -#define AT_RANDOM 25 /* Address of 16 random bytes. */ - -#define AT_EXECFN 31 /* Filename of executable. */ - -/* Pointer to the global system page used for system calls and other - nice things. */ -#define AT_SYSINFO 32 -#define AT_SYSINFO_EHDR 33 - -/* Shapes of the caches. Bits 0-3 contains associativity; bits 4-7 contains - log2 of line size; mask those to get cache size. */ -#define AT_L1I_CACHESHAPE 34 -#define AT_L1D_CACHESHAPE 35 -#define AT_L2_CACHESHAPE 36 -#define AT_L3_CACHESHAPE 37 - -/* Note section contents. Each entry in the note section begins with - a header of a fixed form. */ - -typedef struct -{ - Elf32_Word n_namesz; /* Length of the note's name. */ - Elf32_Word n_descsz; /* Length of the note's descriptor. */ - Elf32_Word n_type; /* Type of the note. */ -} Elf32_Nhdr; - -typedef struct -{ - Elf64_Word n_namesz; /* Length of the note's name. */ - Elf64_Word n_descsz; /* Length of the note's descriptor. */ - Elf64_Word n_type; /* Type of the note. */ -} Elf64_Nhdr; - -/* Known names of notes. */ - -/* Solaris entries in the note section have this name. */ -#define ELF_NOTE_SOLARIS "SUNW Solaris" - -/* Note entries for GNU systems have this name. */ -#define ELF_NOTE_GNU "GNU" - - -/* Defined types of notes for Solaris. */ - -/* Value of descriptor (one word) is desired pagesize for the binary. */ -#define ELF_NOTE_PAGESIZE_HINT 1 - - -/* Defined note types for GNU systems. */ - -/* ABI information. The descriptor consists of words: - word 0: OS descriptor - word 1: major version of the ABI - word 2: minor version of the ABI - word 3: subminor version of the ABI -*/ -#define NT_GNU_ABI_TAG 1 -#define ELF_NOTE_ABI NT_GNU_ABI_TAG /* Old name. */ - -/* Known OSes. These values can appear in word 0 of an - NT_GNU_ABI_TAG note section entry. */ -#define ELF_NOTE_OS_LINUX 0 -#define ELF_NOTE_OS_GNU 1 -#define ELF_NOTE_OS_SOLARIS2 2 -#define ELF_NOTE_OS_FREEBSD 3 - -/* Synthetic hwcap information. The descriptor begins with two words: - word 0: number of entries - word 1: bitmask of enabled entries - Then follow variable-length entries, one byte followed by a - '\0'-terminated hwcap name string. The byte gives the bit - number to test if enabled, (1U << bit) & bitmask. */ -#define NT_GNU_HWCAP 2 - -/* Build ID bits as generated by ld --build-id. - The descriptor consists of any nonzero number of bytes. */ -#define NT_GNU_BUILD_ID 3 - -/* Version note generated by GNU gold containing a version string. */ -#define NT_GNU_GOLD_VERSION 4 - - -/* Move records. */ -typedef struct -{ - Elf32_Xword m_value; /* Symbol value. */ - Elf32_Word m_info; /* Size and index. */ - Elf32_Word m_poffset; /* Symbol offset. */ - Elf32_Half m_repeat; /* Repeat count. */ - Elf32_Half m_stride; /* Stride info. */ -} Elf32_Move; - -typedef struct -{ - Elf64_Xword m_value; /* Symbol value. */ - Elf64_Xword m_info; /* Size and index. */ - Elf64_Xword m_poffset; /* Symbol offset. */ - Elf64_Half m_repeat; /* Repeat count. */ - Elf64_Half m_stride; /* Stride info. */ -} Elf64_Move; - -/* Macro to construct move records. */ -#define ELF32_M_SYM(info) ((info) >> 8) -#define ELF32_M_SIZE(info) ((unsigned char) (info)) -#define ELF32_M_INFO(sym, size) (((sym) << 8) + (unsigned char) (size)) - -#define ELF64_M_SYM(info) ELF32_M_SYM (info) -#define ELF64_M_SIZE(info) ELF32_M_SIZE (info) -#define ELF64_M_INFO(sym, size) ELF32_M_INFO (sym, size) - - -/* Motorola 68k specific definitions. */ - -/* Values for Elf32_Ehdr.e_flags. */ -#define EF_CPU32 0x00810000 - -/* m68k relocs. */ - -#define R_68K_NONE 0 /* No reloc */ -#define R_68K_32 1 /* Direct 32 bit */ -#define R_68K_16 2 /* Direct 16 bit */ -#define R_68K_8 3 /* Direct 8 bit */ -#define R_68K_PC32 4 /* PC relative 32 bit */ -#define R_68K_PC16 5 /* PC relative 16 bit */ -#define R_68K_PC8 6 /* PC relative 8 bit */ -#define R_68K_GOT32 7 /* 32 bit PC relative GOT entry */ -#define R_68K_GOT16 8 /* 16 bit PC relative GOT entry */ -#define R_68K_GOT8 9 /* 8 bit PC relative GOT entry */ -#define R_68K_GOT32O 10 /* 32 bit GOT offset */ -#define R_68K_GOT16O 11 /* 16 bit GOT offset */ -#define R_68K_GOT8O 12 /* 8 bit GOT offset */ -#define R_68K_PLT32 13 /* 32 bit PC relative PLT address */ -#define R_68K_PLT16 14 /* 16 bit PC relative PLT address */ -#define R_68K_PLT8 15 /* 8 bit PC relative PLT address */ -#define R_68K_PLT32O 16 /* 32 bit PLT offset */ -#define R_68K_PLT16O 17 /* 16 bit PLT offset */ -#define R_68K_PLT8O 18 /* 8 bit PLT offset */ -#define R_68K_COPY 19 /* Copy symbol at runtime */ -#define R_68K_GLOB_DAT 20 /* Create GOT entry */ -#define R_68K_JMP_SLOT 21 /* Create PLT entry */ -#define R_68K_RELATIVE 22 /* Adjust by program base */ -#define R_68K_TLS_GD32 25 /* 32 bit GOT offset for GD */ -#define R_68K_TLS_GD16 26 /* 16 bit GOT offset for GD */ -#define R_68K_TLS_GD8 27 /* 8 bit GOT offset for GD */ -#define R_68K_TLS_LDM32 28 /* 32 bit GOT offset for LDM */ -#define R_68K_TLS_LDM16 29 /* 16 bit GOT offset for LDM */ -#define R_68K_TLS_LDM8 30 /* 8 bit GOT offset for LDM */ -#define R_68K_TLS_LDO32 31 /* 32 bit module-relative offset */ -#define R_68K_TLS_LDO16 32 /* 16 bit module-relative offset */ -#define R_68K_TLS_LDO8 33 /* 8 bit module-relative offset */ -#define R_68K_TLS_IE32 34 /* 32 bit GOT offset for IE */ -#define R_68K_TLS_IE16 35 /* 16 bit GOT offset for IE */ -#define R_68K_TLS_IE8 36 /* 8 bit GOT offset for IE */ -#define R_68K_TLS_LE32 37 /* 32 bit offset relative to - static TLS block */ -#define R_68K_TLS_LE16 38 /* 16 bit offset relative to - static TLS block */ -#define R_68K_TLS_LE8 39 /* 8 bit offset relative to - static TLS block */ -#define R_68K_TLS_DTPMOD32 40 /* 32 bit module number */ -#define R_68K_TLS_DTPREL32 41 /* 32 bit module-relative offset */ -#define R_68K_TLS_TPREL32 42 /* 32 bit TP-relative offset */ -/* Keep this the last entry. */ -#define R_68K_NUM 43 - -/* Intel 80386 specific definitions. */ - -/* i386 relocs. */ - -#define R_386_NONE 0 /* No reloc */ -#define R_386_32 1 /* Direct 32 bit */ -#define R_386_PC32 2 /* PC relative 32 bit */ -#define R_386_GOT32 3 /* 32 bit GOT entry */ -#define R_386_PLT32 4 /* 32 bit PLT address */ -#define R_386_COPY 5 /* Copy symbol at runtime */ -#define R_386_GLOB_DAT 6 /* Create GOT entry */ -#define R_386_JMP_SLOT 7 /* Create PLT entry */ -#define R_386_RELATIVE 8 /* Adjust by program base */ -#define R_386_GOTOFF 9 /* 32 bit offset to GOT */ -#define R_386_GOTPC 10 /* 32 bit PC relative offset to GOT */ -#define R_386_32PLT 11 -#define R_386_TLS_TPOFF 14 /* Offset in static TLS block */ -#define R_386_TLS_IE 15 /* Address of GOT entry for static TLS - block offset */ -#define R_386_TLS_GOTIE 16 /* GOT entry for static TLS block - offset */ -#define R_386_TLS_LE 17 /* Offset relative to static TLS - block */ -#define R_386_TLS_GD 18 /* Direct 32 bit for GNU version of - general dynamic thread local data */ -#define R_386_TLS_LDM 19 /* Direct 32 bit for GNU version of - local dynamic thread local data - in LE code */ -#define R_386_16 20 -#define R_386_PC16 21 -#define R_386_8 22 -#define R_386_PC8 23 -#define R_386_TLS_GD_32 24 /* Direct 32 bit for general dynamic - thread local data */ -#define R_386_TLS_GD_PUSH 25 /* Tag for pushl in GD TLS code */ -#define R_386_TLS_GD_CALL 26 /* Relocation for call to - __tls_get_addr() */ -#define R_386_TLS_GD_POP 27 /* Tag for popl in GD TLS code */ -#define R_386_TLS_LDM_32 28 /* Direct 32 bit for local dynamic - thread local data in LE code */ -#define R_386_TLS_LDM_PUSH 29 /* Tag for pushl in LDM TLS code */ -#define R_386_TLS_LDM_CALL 30 /* Relocation for call to - __tls_get_addr() in LDM code */ -#define R_386_TLS_LDM_POP 31 /* Tag for popl in LDM TLS code */ -#define R_386_TLS_LDO_32 32 /* Offset relative to TLS block */ -#define R_386_TLS_IE_32 33 /* GOT entry for negated static TLS - block offset */ -#define R_386_TLS_LE_32 34 /* Negated offset relative to static - TLS block */ -#define R_386_TLS_DTPMOD32 35 /* ID of module containing symbol */ -#define R_386_TLS_DTPOFF32 36 /* Offset in TLS block */ -#define R_386_TLS_TPOFF32 37 /* Negated offset in static TLS block */ -/* 38? */ -#define R_386_TLS_GOTDESC 39 /* GOT offset for TLS descriptor. */ -#define R_386_TLS_DESC_CALL 40 /* Marker of call through TLS - descriptor for - relaxation. */ -#define R_386_TLS_DESC 41 /* TLS descriptor containing - pointer to code and to - argument, returning the TLS - offset for the symbol. */ -#define R_386_IRELATIVE 42 /* Adjust indirectly by program base */ -/* Keep this the last entry. */ -#define R_386_NUM 43 - -/* SUN SPARC specific definitions. */ - -/* Legal values for ST_TYPE subfield of st_info (symbol type). */ - -#define STT_SPARC_REGISTER 13 /* Global register reserved to app. */ - -/* Values for Elf64_Ehdr.e_flags. */ - -#define EF_SPARCV9_MM 3 -#define EF_SPARCV9_TSO 0 -#define EF_SPARCV9_PSO 1 -#define EF_SPARCV9_RMO 2 -#define EF_SPARC_LEDATA 0x800000 /* little endian data */ -#define EF_SPARC_EXT_MASK 0xFFFF00 -#define EF_SPARC_32PLUS 0x000100 /* generic V8+ features */ -#define EF_SPARC_SUN_US1 0x000200 /* Sun UltraSPARC1 extensions */ -#define EF_SPARC_HAL_R1 0x000400 /* HAL R1 extensions */ -#define EF_SPARC_SUN_US3 0x000800 /* Sun UltraSPARCIII extensions */ - -/* SPARC relocs. */ - -#define R_SPARC_NONE 0 /* No reloc */ -#define R_SPARC_8 1 /* Direct 8 bit */ -#define R_SPARC_16 2 /* Direct 16 bit */ -#define R_SPARC_32 3 /* Direct 32 bit */ -#define R_SPARC_DISP8 4 /* PC relative 8 bit */ -#define R_SPARC_DISP16 5 /* PC relative 16 bit */ -#define R_SPARC_DISP32 6 /* PC relative 32 bit */ -#define R_SPARC_WDISP30 7 /* PC relative 30 bit shifted */ -#define R_SPARC_WDISP22 8 /* PC relative 22 bit shifted */ -#define R_SPARC_HI22 9 /* High 22 bit */ -#define R_SPARC_22 10 /* Direct 22 bit */ -#define R_SPARC_13 11 /* Direct 13 bit */ -#define R_SPARC_LO10 12 /* Truncated 10 bit */ -#define R_SPARC_GOT10 13 /* Truncated 10 bit GOT entry */ -#define R_SPARC_GOT13 14 /* 13 bit GOT entry */ -#define R_SPARC_GOT22 15 /* 22 bit GOT entry shifted */ -#define R_SPARC_PC10 16 /* PC relative 10 bit truncated */ -#define R_SPARC_PC22 17 /* PC relative 22 bit shifted */ -#define R_SPARC_WPLT30 18 /* 30 bit PC relative PLT address */ -#define R_SPARC_COPY 19 /* Copy symbol at runtime */ -#define R_SPARC_GLOB_DAT 20 /* Create GOT entry */ -#define R_SPARC_JMP_SLOT 21 /* Create PLT entry */ -#define R_SPARC_RELATIVE 22 /* Adjust by program base */ -#define R_SPARC_UA32 23 /* Direct 32 bit unaligned */ - -/* Additional Sparc64 relocs. */ - -#define R_SPARC_PLT32 24 /* Direct 32 bit ref to PLT entry */ -#define R_SPARC_HIPLT22 25 /* High 22 bit PLT entry */ -#define R_SPARC_LOPLT10 26 /* Truncated 10 bit PLT entry */ -#define R_SPARC_PCPLT32 27 /* PC rel 32 bit ref to PLT entry */ -#define R_SPARC_PCPLT22 28 /* PC rel high 22 bit PLT entry */ -#define R_SPARC_PCPLT10 29 /* PC rel trunc 10 bit PLT entry */ -#define R_SPARC_10 30 /* Direct 10 bit */ -#define R_SPARC_11 31 /* Direct 11 bit */ -#define R_SPARC_64 32 /* Direct 64 bit */ -#define R_SPARC_OLO10 33 /* 10bit with secondary 13bit addend */ -#define R_SPARC_HH22 34 /* Top 22 bits of direct 64 bit */ -#define R_SPARC_HM10 35 /* High middle 10 bits of ... */ -#define R_SPARC_LM22 36 /* Low middle 22 bits of ... */ -#define R_SPARC_PC_HH22 37 /* Top 22 bits of pc rel 64 bit */ -#define R_SPARC_PC_HM10 38 /* High middle 10 bit of ... */ -#define R_SPARC_PC_LM22 39 /* Low miggle 22 bits of ... */ -#define R_SPARC_WDISP16 40 /* PC relative 16 bit shifted */ -#define R_SPARC_WDISP19 41 /* PC relative 19 bit shifted */ -#define R_SPARC_GLOB_JMP 42 /* was part of v9 ABI but was removed */ -#define R_SPARC_7 43 /* Direct 7 bit */ -#define R_SPARC_5 44 /* Direct 5 bit */ -#define R_SPARC_6 45 /* Direct 6 bit */ -#define R_SPARC_DISP64 46 /* PC relative 64 bit */ -#define R_SPARC_PLT64 47 /* Direct 64 bit ref to PLT entry */ -#define R_SPARC_HIX22 48 /* High 22 bit complemented */ -#define R_SPARC_LOX10 49 /* Truncated 11 bit complemented */ -#define R_SPARC_H44 50 /* Direct high 12 of 44 bit */ -#define R_SPARC_M44 51 /* Direct mid 22 of 44 bit */ -#define R_SPARC_L44 52 /* Direct low 10 of 44 bit */ -#define R_SPARC_REGISTER 53 /* Global register usage */ -#define R_SPARC_UA64 54 /* Direct 64 bit unaligned */ -#define R_SPARC_UA16 55 /* Direct 16 bit unaligned */ -#define R_SPARC_TLS_GD_HI22 56 -#define R_SPARC_TLS_GD_LO10 57 -#define R_SPARC_TLS_GD_ADD 58 -#define R_SPARC_TLS_GD_CALL 59 -#define R_SPARC_TLS_LDM_HI22 60 -#define R_SPARC_TLS_LDM_LO10 61 -#define R_SPARC_TLS_LDM_ADD 62 -#define R_SPARC_TLS_LDM_CALL 63 -#define R_SPARC_TLS_LDO_HIX22 64 -#define R_SPARC_TLS_LDO_LOX10 65 -#define R_SPARC_TLS_LDO_ADD 66 -#define R_SPARC_TLS_IE_HI22 67 -#define R_SPARC_TLS_IE_LO10 68 -#define R_SPARC_TLS_IE_LD 69 -#define R_SPARC_TLS_IE_LDX 70 -#define R_SPARC_TLS_IE_ADD 71 -#define R_SPARC_TLS_LE_HIX22 72 -#define R_SPARC_TLS_LE_LOX10 73 -#define R_SPARC_TLS_DTPMOD32 74 -#define R_SPARC_TLS_DTPMOD64 75 -#define R_SPARC_TLS_DTPOFF32 76 -#define R_SPARC_TLS_DTPOFF64 77 -#define R_SPARC_TLS_TPOFF32 78 -#define R_SPARC_TLS_TPOFF64 79 -#define R_SPARC_GOTDATA_HIX22 80 -#define R_SPARC_GOTDATA_LOX10 81 -#define R_SPARC_GOTDATA_OP_HIX22 82 -#define R_SPARC_GOTDATA_OP_LOX10 83 -#define R_SPARC_GOTDATA_OP 84 -#define R_SPARC_H34 85 -#define R_SPARC_SIZE32 86 -#define R_SPARC_SIZE64 87 -#define R_SPARC_JMP_IREL 248 -#define R_SPARC_IRELATIVE 249 -#define R_SPARC_GNU_VTINHERIT 250 -#define R_SPARC_GNU_VTENTRY 251 -#define R_SPARC_REV32 252 -/* Keep this the last entry. */ -#define R_SPARC_NUM 253 - -/* For Sparc64, legal values for d_tag of Elf64_Dyn. */ - -#define DT_SPARC_REGISTER 0x70000001 -#define DT_SPARC_NUM 2 - -/* MIPS R3000 specific definitions. */ - -/* Legal values for e_flags field of Elf32_Ehdr. */ - -#define EF_MIPS_NOREORDER 1 /* A .noreorder directive was used */ -#define EF_MIPS_PIC 2 /* Contains PIC code */ -#define EF_MIPS_CPIC 4 /* Uses PIC calling sequence */ -#define EF_MIPS_XGOT 8 -#define EF_MIPS_64BIT_WHIRL 16 -#define EF_MIPS_ABI2 32 -#define EF_MIPS_ABI_ON32 64 -#define EF_MIPS_ARCH 0xf0000000 /* MIPS architecture level */ - -/* Legal values for MIPS architecture level. */ - -#define EF_MIPS_ARCH_1 0x00000000 /* -mips1 code. */ -#define EF_MIPS_ARCH_2 0x10000000 /* -mips2 code. */ -#define EF_MIPS_ARCH_3 0x20000000 /* -mips3 code. */ -#define EF_MIPS_ARCH_4 0x30000000 /* -mips4 code. */ -#define EF_MIPS_ARCH_5 0x40000000 /* -mips5 code. */ -#define EF_MIPS_ARCH_32 0x60000000 /* MIPS32 code. */ -#define EF_MIPS_ARCH_64 0x70000000 /* MIPS64 code. */ - -/* The following are non-official names and should not be used. */ - -#define E_MIPS_ARCH_1 0x00000000 /* -mips1 code. */ -#define E_MIPS_ARCH_2 0x10000000 /* -mips2 code. */ -#define E_MIPS_ARCH_3 0x20000000 /* -mips3 code. */ -#define E_MIPS_ARCH_4 0x30000000 /* -mips4 code. */ -#define E_MIPS_ARCH_5 0x40000000 /* -mips5 code. */ -#define E_MIPS_ARCH_32 0x60000000 /* MIPS32 code. */ -#define E_MIPS_ARCH_64 0x70000000 /* MIPS64 code. */ - -/* Special section indices. */ - -#define SHN_MIPS_ACOMMON 0xff00 /* Allocated common symbols */ -#define SHN_MIPS_TEXT 0xff01 /* Allocated test symbols. */ -#define SHN_MIPS_DATA 0xff02 /* Allocated data symbols. */ -#define SHN_MIPS_SCOMMON 0xff03 /* Small common symbols */ -#define SHN_MIPS_SUNDEFINED 0xff04 /* Small undefined symbols */ - -/* Legal values for sh_type field of Elf32_Shdr. */ - -#define SHT_MIPS_LIBLIST 0x70000000 /* Shared objects used in link */ -#define SHT_MIPS_MSYM 0x70000001 -#define SHT_MIPS_CONFLICT 0x70000002 /* Conflicting symbols */ -#define SHT_MIPS_GPTAB 0x70000003 /* Global data area sizes */ -#define SHT_MIPS_UCODE 0x70000004 /* Reserved for SGI/MIPS compilers */ -#define SHT_MIPS_DEBUG 0x70000005 /* MIPS ECOFF debugging information*/ -#define SHT_MIPS_REGINFO 0x70000006 /* Register usage information */ -#define SHT_MIPS_PACKAGE 0x70000007 -#define SHT_MIPS_PACKSYM 0x70000008 -#define SHT_MIPS_RELD 0x70000009 -#define SHT_MIPS_IFACE 0x7000000b -#define SHT_MIPS_CONTENT 0x7000000c -#define SHT_MIPS_OPTIONS 0x7000000d /* Miscellaneous options. */ -#define SHT_MIPS_SHDR 0x70000010 -#define SHT_MIPS_FDESC 0x70000011 -#define SHT_MIPS_EXTSYM 0x70000012 -#define SHT_MIPS_DENSE 0x70000013 -#define SHT_MIPS_PDESC 0x70000014 -#define SHT_MIPS_LOCSYM 0x70000015 -#define SHT_MIPS_AUXSYM 0x70000016 -#define SHT_MIPS_OPTSYM 0x70000017 -#define SHT_MIPS_LOCSTR 0x70000018 -#define SHT_MIPS_LINE 0x70000019 -#define SHT_MIPS_RFDESC 0x7000001a -#define SHT_MIPS_DELTASYM 0x7000001b -#define SHT_MIPS_DELTAINST 0x7000001c -#define SHT_MIPS_DELTACLASS 0x7000001d -#define SHT_MIPS_DWARF 0x7000001e /* DWARF debugging information. */ -#define SHT_MIPS_DELTADECL 0x7000001f -#define SHT_MIPS_SYMBOL_LIB 0x70000020 -#define SHT_MIPS_EVENTS 0x70000021 /* Event section. */ -#define SHT_MIPS_TRANSLATE 0x70000022 -#define SHT_MIPS_PIXIE 0x70000023 -#define SHT_MIPS_XLATE 0x70000024 -#define SHT_MIPS_XLATE_DEBUG 0x70000025 -#define SHT_MIPS_WHIRL 0x70000026 -#define SHT_MIPS_EH_REGION 0x70000027 -#define SHT_MIPS_XLATE_OLD 0x70000028 -#define SHT_MIPS_PDR_EXCEPTION 0x70000029 - -/* Legal values for sh_flags field of Elf32_Shdr. */ - -#define SHF_MIPS_GPREL 0x10000000 /* Must be part of global data area */ -#define SHF_MIPS_MERGE 0x20000000 -#define SHF_MIPS_ADDR 0x40000000 -#define SHF_MIPS_STRINGS 0x80000000 -#define SHF_MIPS_NOSTRIP 0x08000000 -#define SHF_MIPS_LOCAL 0x04000000 -#define SHF_MIPS_NAMES 0x02000000 -#define SHF_MIPS_NODUPE 0x01000000 - - -/* Symbol tables. */ - -/* MIPS specific values for `st_other'. */ -#define STO_MIPS_DEFAULT 0x0 -#define STO_MIPS_INTERNAL 0x1 -#define STO_MIPS_HIDDEN 0x2 -#define STO_MIPS_PROTECTED 0x3 -#define STO_MIPS_PLT 0x8 -#define STO_MIPS_SC_ALIGN_UNUSED 0xff - -/* MIPS specific values for `st_info'. */ -#define STB_MIPS_SPLIT_COMMON 13 - -/* Entries found in sections of type SHT_MIPS_GPTAB. */ - -typedef union -{ - struct - { - Elf32_Word gt_current_g_value; /* -G value used for compilation */ - Elf32_Word gt_unused; /* Not used */ - } gt_header; /* First entry in section */ - struct - { - Elf32_Word gt_g_value; /* If this value were used for -G */ - Elf32_Word gt_bytes; /* This many bytes would be used */ - } gt_entry; /* Subsequent entries in section */ -} Elf32_gptab; - -/* Entry found in sections of type SHT_MIPS_REGINFO. */ - -typedef struct -{ - Elf32_Word ri_gprmask; /* General registers used */ - Elf32_Word ri_cprmask[4]; /* Coprocessor registers used */ - Elf32_Sword ri_gp_value; /* $gp register value */ -} Elf32_RegInfo; - -/* Entries found in sections of type SHT_MIPS_OPTIONS. */ - -typedef struct -{ - unsigned char kind; /* Determines interpretation of the - variable part of descriptor. */ - unsigned char size; /* Size of descriptor, including header. */ - Elf32_Section section; /* Section header index of section affected, - 0 for global options. */ - Elf32_Word info; /* Kind-specific information. */ -} Elf_Options; - -/* Values for `kind' field in Elf_Options. */ - -#define ODK_NULL 0 /* Undefined. */ -#define ODK_REGINFO 1 /* Register usage information. */ -#define ODK_EXCEPTIONS 2 /* Exception processing options. */ -#define ODK_PAD 3 /* Section padding options. */ -#define ODK_HWPATCH 4 /* Hardware workarounds performed */ -#define ODK_FILL 5 /* record the fill value used by the linker. */ -#define ODK_TAGS 6 /* reserve space for desktop tools to write. */ -#define ODK_HWAND 7 /* HW workarounds. 'AND' bits when merging. */ -#define ODK_HWOR 8 /* HW workarounds. 'OR' bits when merging. */ - -/* Values for `info' in Elf_Options for ODK_EXCEPTIONS entries. */ - -#define OEX_FPU_MIN 0x1f /* FPE's which MUST be enabled. */ -#define OEX_FPU_MAX 0x1f00 /* FPE's which MAY be enabled. */ -#define OEX_PAGE0 0x10000 /* page zero must be mapped. */ -#define OEX_SMM 0x20000 /* Force sequential memory mode? */ -#define OEX_FPDBUG 0x40000 /* Force floating point debug mode? */ -#define OEX_PRECISEFP OEX_FPDBUG -#define OEX_DISMISS 0x80000 /* Dismiss invalid address faults? */ - -#define OEX_FPU_INVAL 0x10 -#define OEX_FPU_DIV0 0x08 -#define OEX_FPU_OFLO 0x04 -#define OEX_FPU_UFLO 0x02 -#define OEX_FPU_INEX 0x01 - -/* Masks for `info' in Elf_Options for an ODK_HWPATCH entry. */ - -#define OHW_R4KEOP 0x1 /* R4000 end-of-page patch. */ -#define OHW_R8KPFETCH 0x2 /* may need R8000 prefetch patch. */ -#define OHW_R5KEOP 0x4 /* R5000 end-of-page patch. */ -#define OHW_R5KCVTL 0x8 /* R5000 cvt.[ds].l bug. clean=1. */ - -#define OPAD_PREFIX 0x1 -#define OPAD_POSTFIX 0x2 -#define OPAD_SYMBOL 0x4 - -/* Entry found in `.options' section. */ - -typedef struct -{ - Elf32_Word hwp_flags1; /* Extra flags. */ - Elf32_Word hwp_flags2; /* Extra flags. */ -} Elf_Options_Hw; - -/* Masks for `info' in ElfOptions for ODK_HWAND and ODK_HWOR entries. */ - -#define OHWA0_R4KEOP_CHECKED 0x00000001 -#define OHWA1_R4KEOP_CLEAN 0x00000002 - -/* MIPS relocs. */ - -#define R_MIPS_NONE 0 /* No reloc */ -#define R_MIPS_16 1 /* Direct 16 bit */ -#define R_MIPS_32 2 /* Direct 32 bit */ -#define R_MIPS_REL32 3 /* PC relative 32 bit */ -#define R_MIPS_26 4 /* Direct 26 bit shifted */ -#define R_MIPS_HI16 5 /* High 16 bit */ -#define R_MIPS_LO16 6 /* Low 16 bit */ -#define R_MIPS_GPREL16 7 /* GP relative 16 bit */ -#define R_MIPS_LITERAL 8 /* 16 bit literal entry */ -#define R_MIPS_GOT16 9 /* 16 bit GOT entry */ -#define R_MIPS_PC16 10 /* PC relative 16 bit */ -#define R_MIPS_CALL16 11 /* 16 bit GOT entry for function */ -#define R_MIPS_GPREL32 12 /* GP relative 32 bit */ - -#define R_MIPS_SHIFT5 16 -#define R_MIPS_SHIFT6 17 -#define R_MIPS_64 18 -#define R_MIPS_GOT_DISP 19 -#define R_MIPS_GOT_PAGE 20 -#define R_MIPS_GOT_OFST 21 -#define R_MIPS_GOT_HI16 22 -#define R_MIPS_GOT_LO16 23 -#define R_MIPS_SUB 24 -#define R_MIPS_INSERT_A 25 -#define R_MIPS_INSERT_B 26 -#define R_MIPS_DELETE 27 -#define R_MIPS_HIGHER 28 -#define R_MIPS_HIGHEST 29 -#define R_MIPS_CALL_HI16 30 -#define R_MIPS_CALL_LO16 31 -#define R_MIPS_SCN_DISP 32 -#define R_MIPS_REL16 33 -#define R_MIPS_ADD_IMMEDIATE 34 -#define R_MIPS_PJUMP 35 -#define R_MIPS_RELGOT 36 -#define R_MIPS_JALR 37 -#define R_MIPS_TLS_DTPMOD32 38 /* Module number 32 bit */ -#define R_MIPS_TLS_DTPREL32 39 /* Module-relative offset 32 bit */ -#define R_MIPS_TLS_DTPMOD64 40 /* Module number 64 bit */ -#define R_MIPS_TLS_DTPREL64 41 /* Module-relative offset 64 bit */ -#define R_MIPS_TLS_GD 42 /* 16 bit GOT offset for GD */ -#define R_MIPS_TLS_LDM 43 /* 16 bit GOT offset for LDM */ -#define R_MIPS_TLS_DTPREL_HI16 44 /* Module-relative offset, high 16 bits */ -#define R_MIPS_TLS_DTPREL_LO16 45 /* Module-relative offset, low 16 bits */ -#define R_MIPS_TLS_GOTTPREL 46 /* 16 bit GOT offset for IE */ -#define R_MIPS_TLS_TPREL32 47 /* TP-relative offset, 32 bit */ -#define R_MIPS_TLS_TPREL64 48 /* TP-relative offset, 64 bit */ -#define R_MIPS_TLS_TPREL_HI16 49 /* TP-relative offset, high 16 bits */ -#define R_MIPS_TLS_TPREL_LO16 50 /* TP-relative offset, low 16 bits */ -#define R_MIPS_GLOB_DAT 51 -#define R_MIPS_COPY 126 -#define R_MIPS_JUMP_SLOT 127 -/* Keep this the last entry. */ -#define R_MIPS_NUM 128 - -/* Legal values for p_type field of Elf32_Phdr. */ - -#define PT_MIPS_REGINFO 0x70000000 /* Register usage information */ -#define PT_MIPS_RTPROC 0x70000001 /* Runtime procedure table. */ -#define PT_MIPS_OPTIONS 0x70000002 - -/* Special program header types. */ - -#define PF_MIPS_LOCAL 0x10000000 - -/* Legal values for d_tag field of Elf32_Dyn. */ - -#define DT_MIPS_RLD_VERSION 0x70000001 /* Runtime linker interface version */ -#define DT_MIPS_TIME_STAMP 0x70000002 /* Timestamp */ -#define DT_MIPS_ICHECKSUM 0x70000003 /* Checksum */ -#define DT_MIPS_IVERSION 0x70000004 /* Version string (string tbl index) */ -#define DT_MIPS_FLAGS 0x70000005 /* Flags */ -#define DT_MIPS_BASE_ADDRESS 0x70000006 /* Base address */ -#define DT_MIPS_MSYM 0x70000007 -#define DT_MIPS_CONFLICT 0x70000008 /* Address of CONFLICT section */ -#define DT_MIPS_LIBLIST 0x70000009 /* Address of LIBLIST section */ -#define DT_MIPS_LOCAL_GOTNO 0x7000000a /* Number of local GOT entries */ -#define DT_MIPS_CONFLICTNO 0x7000000b /* Number of CONFLICT entries */ -#define DT_MIPS_LIBLISTNO 0x70000010 /* Number of LIBLIST entries */ -#define DT_MIPS_SYMTABNO 0x70000011 /* Number of DYNSYM entries */ -#define DT_MIPS_UNREFEXTNO 0x70000012 /* First external DYNSYM */ -#define DT_MIPS_GOTSYM 0x70000013 /* First GOT entry in DYNSYM */ -#define DT_MIPS_HIPAGENO 0x70000014 /* Number of GOT page table entries */ -#define DT_MIPS_RLD_MAP 0x70000016 /* Address of run time loader map. */ -#define DT_MIPS_DELTA_CLASS 0x70000017 /* Delta C++ class definition. */ -#define DT_MIPS_DELTA_CLASS_NO 0x70000018 /* Number of entries in - DT_MIPS_DELTA_CLASS. */ -#define DT_MIPS_DELTA_INSTANCE 0x70000019 /* Delta C++ class instances. */ -#define DT_MIPS_DELTA_INSTANCE_NO 0x7000001a /* Number of entries in - DT_MIPS_DELTA_INSTANCE. */ -#define DT_MIPS_DELTA_RELOC 0x7000001b /* Delta relocations. */ -#define DT_MIPS_DELTA_RELOC_NO 0x7000001c /* Number of entries in - DT_MIPS_DELTA_RELOC. */ -#define DT_MIPS_DELTA_SYM 0x7000001d /* Delta symbols that Delta - relocations refer to. */ -#define DT_MIPS_DELTA_SYM_NO 0x7000001e /* Number of entries in - DT_MIPS_DELTA_SYM. */ -#define DT_MIPS_DELTA_CLASSSYM 0x70000020 /* Delta symbols that hold the - class declaration. */ -#define DT_MIPS_DELTA_CLASSSYM_NO 0x70000021 /* Number of entries in - DT_MIPS_DELTA_CLASSSYM. */ -#define DT_MIPS_CXX_FLAGS 0x70000022 /* Flags indicating for C++ flavor. */ -#define DT_MIPS_PIXIE_INIT 0x70000023 -#define DT_MIPS_SYMBOL_LIB 0x70000024 -#define DT_MIPS_LOCALPAGE_GOTIDX 0x70000025 -#define DT_MIPS_LOCAL_GOTIDX 0x70000026 -#define DT_MIPS_HIDDEN_GOTIDX 0x70000027 -#define DT_MIPS_PROTECTED_GOTIDX 0x70000028 -#define DT_MIPS_OPTIONS 0x70000029 /* Address of .options. */ -#define DT_MIPS_INTERFACE 0x7000002a /* Address of .interface. */ -#define DT_MIPS_DYNSTR_ALIGN 0x7000002b -#define DT_MIPS_INTERFACE_SIZE 0x7000002c /* Size of the .interface section. */ -#define DT_MIPS_RLD_TEXT_RESOLVE_ADDR 0x7000002d /* Address of rld_text_rsolve - function stored in GOT. */ -#define DT_MIPS_PERF_SUFFIX 0x7000002e /* Default suffix of dso to be added - by rld on dlopen() calls. */ -#define DT_MIPS_COMPACT_SIZE 0x7000002f /* (O32)Size of compact rel section. */ -#define DT_MIPS_GP_VALUE 0x70000030 /* GP value for aux GOTs. */ -#define DT_MIPS_AUX_DYNAMIC 0x70000031 /* Address of aux .dynamic. */ -/* The address of .got.plt in an executable using the new non-PIC ABI. */ -#define DT_MIPS_PLTGOT 0x70000032 -/* The base of the PLT in an executable using the new non-PIC ABI if that - PLT is writable. For a non-writable PLT, this is omitted or has a zero - value. */ -#define DT_MIPS_RWPLT 0x70000034 -#define DT_MIPS_NUM 0x35 - -/* Legal values for DT_MIPS_FLAGS Elf32_Dyn entry. */ - -#define RHF_NONE 0 /* No flags */ -#define RHF_QUICKSTART (1 << 0) /* Use quickstart */ -#define RHF_NOTPOT (1 << 1) /* Hash size not power of 2 */ -#define RHF_NO_LIBRARY_REPLACEMENT (1 << 2) /* Ignore LD_LIBRARY_PATH */ -#define RHF_NO_MOVE (1 << 3) -#define RHF_SGI_ONLY (1 << 4) -#define RHF_GUARANTEE_INIT (1 << 5) -#define RHF_DELTA_C_PLUS_PLUS (1 << 6) -#define RHF_GUARANTEE_START_INIT (1 << 7) -#define RHF_PIXIE (1 << 8) -#define RHF_DEFAULT_DELAY_LOAD (1 << 9) -#define RHF_REQUICKSTART (1 << 10) -#define RHF_REQUICKSTARTED (1 << 11) -#define RHF_CORD (1 << 12) -#define RHF_NO_UNRES_UNDEF (1 << 13) -#define RHF_RLD_ORDER_SAFE (1 << 14) - -/* Entries found in sections of type SHT_MIPS_LIBLIST. */ - -typedef struct -{ - Elf32_Word l_name; /* Name (string table index) */ - Elf32_Word l_time_stamp; /* Timestamp */ - Elf32_Word l_checksum; /* Checksum */ - Elf32_Word l_version; /* Interface version */ - Elf32_Word l_flags; /* Flags */ -} Elf32_Lib; - -typedef struct -{ - Elf64_Word l_name; /* Name (string table index) */ - Elf64_Word l_time_stamp; /* Timestamp */ - Elf64_Word l_checksum; /* Checksum */ - Elf64_Word l_version; /* Interface version */ - Elf64_Word l_flags; /* Flags */ -} Elf64_Lib; - - -/* Legal values for l_flags. */ - -#define LL_NONE 0 -#define LL_EXACT_MATCH (1 << 0) /* Require exact match */ -#define LL_IGNORE_INT_VER (1 << 1) /* Ignore interface version */ -#define LL_REQUIRE_MINOR (1 << 2) -#define LL_EXPORTS (1 << 3) -#define LL_DELAY_LOAD (1 << 4) -#define LL_DELTA (1 << 5) - -/* Entries found in sections of type SHT_MIPS_CONFLICT. */ - -typedef Elf32_Addr Elf32_Conflict; - - -/* HPPA specific definitions. */ - -/* Legal values for e_flags field of Elf32_Ehdr. */ - -#define EF_PARISC_TRAPNIL 0x00010000 /* Trap nil pointer dereference. */ -#define EF_PARISC_EXT 0x00020000 /* Program uses arch. extensions. */ -#define EF_PARISC_LSB 0x00040000 /* Program expects little endian. */ -#define EF_PARISC_WIDE 0x00080000 /* Program expects wide mode. */ -#define EF_PARISC_NO_KABP 0x00100000 /* No kernel assisted branch - prediction. */ -#define EF_PARISC_LAZYSWAP 0x00400000 /* Allow lazy swapping. */ -#define EF_PARISC_ARCH 0x0000ffff /* Architecture version. */ - -/* Defined values for `e_flags & EF_PARISC_ARCH' are: */ - -#define EFA_PARISC_1_0 0x020b /* PA-RISC 1.0 big-endian. */ -#define EFA_PARISC_1_1 0x0210 /* PA-RISC 1.1 big-endian. */ -#define EFA_PARISC_2_0 0x0214 /* PA-RISC 2.0 big-endian. */ - -/* Additional section indeces. */ - -#define SHN_PARISC_ANSI_COMMON 0xff00 /* Section for tenatively declared - symbols in ANSI C. */ -#define SHN_PARISC_HUGE_COMMON 0xff01 /* Common blocks in huge model. */ - -/* Legal values for sh_type field of Elf32_Shdr. */ - -#define SHT_PARISC_EXT 0x70000000 /* Contains product specific ext. */ -#define SHT_PARISC_UNWIND 0x70000001 /* Unwind information. */ -#define SHT_PARISC_DOC 0x70000002 /* Debug info for optimized code. */ - -/* Legal values for sh_flags field of Elf32_Shdr. */ - -#define SHF_PARISC_SHORT 0x20000000 /* Section with short addressing. */ -#define SHF_PARISC_HUGE 0x40000000 /* Section far from gp. */ -#define SHF_PARISC_SBP 0x80000000 /* Static branch prediction code. */ - -/* Legal values for ST_TYPE subfield of st_info (symbol type). */ - -#define STT_PARISC_MILLICODE 13 /* Millicode function entry point. */ - -#define STT_HP_OPAQUE (STT_LOOS + 0x1) -#define STT_HP_STUB (STT_LOOS + 0x2) - -/* HPPA relocs. */ - -#define R_PARISC_NONE 0 /* No reloc. */ -#define R_PARISC_DIR32 1 /* Direct 32-bit reference. */ -#define R_PARISC_DIR21L 2 /* Left 21 bits of eff. address. */ -#define R_PARISC_DIR17R 3 /* Right 17 bits of eff. address. */ -#define R_PARISC_DIR17F 4 /* 17 bits of eff. address. */ -#define R_PARISC_DIR14R 6 /* Right 14 bits of eff. address. */ -#define R_PARISC_PCREL32 9 /* 32-bit rel. address. */ -#define R_PARISC_PCREL21L 10 /* Left 21 bits of rel. address. */ -#define R_PARISC_PCREL17R 11 /* Right 17 bits of rel. address. */ -#define R_PARISC_PCREL17F 12 /* 17 bits of rel. address. */ -#define R_PARISC_PCREL14R 14 /* Right 14 bits of rel. address. */ -#define R_PARISC_DPREL21L 18 /* Left 21 bits of rel. address. */ -#define R_PARISC_DPREL14R 22 /* Right 14 bits of rel. address. */ -#define R_PARISC_GPREL21L 26 /* GP-relative, left 21 bits. */ -#define R_PARISC_GPREL14R 30 /* GP-relative, right 14 bits. */ -#define R_PARISC_LTOFF21L 34 /* LT-relative, left 21 bits. */ -#define R_PARISC_LTOFF14R 38 /* LT-relative, right 14 bits. */ -#define R_PARISC_SECREL32 41 /* 32 bits section rel. address. */ -#define R_PARISC_SEGBASE 48 /* No relocation, set segment base. */ -#define R_PARISC_SEGREL32 49 /* 32 bits segment rel. address. */ -#define R_PARISC_PLTOFF21L 50 /* PLT rel. address, left 21 bits. */ -#define R_PARISC_PLTOFF14R 54 /* PLT rel. address, right 14 bits. */ -#define R_PARISC_LTOFF_FPTR32 57 /* 32 bits LT-rel. function pointer. */ -#define R_PARISC_LTOFF_FPTR21L 58 /* LT-rel. fct ptr, left 21 bits. */ -#define R_PARISC_LTOFF_FPTR14R 62 /* LT-rel. fct ptr, right 14 bits. */ -#define R_PARISC_FPTR64 64 /* 64 bits function address. */ -#define R_PARISC_PLABEL32 65 /* 32 bits function address. */ -#define R_PARISC_PLABEL21L 66 /* Left 21 bits of fdesc address. */ -#define R_PARISC_PLABEL14R 70 /* Right 14 bits of fdesc address. */ -#define R_PARISC_PCREL64 72 /* 64 bits PC-rel. address. */ -#define R_PARISC_PCREL22F 74 /* 22 bits PC-rel. address. */ -#define R_PARISC_PCREL14WR 75 /* PC-rel. address, right 14 bits. */ -#define R_PARISC_PCREL14DR 76 /* PC rel. address, right 14 bits. */ -#define R_PARISC_PCREL16F 77 /* 16 bits PC-rel. address. */ -#define R_PARISC_PCREL16WF 78 /* 16 bits PC-rel. address. */ -#define R_PARISC_PCREL16DF 79 /* 16 bits PC-rel. address. */ -#define R_PARISC_DIR64 80 /* 64 bits of eff. address. */ -#define R_PARISC_DIR14WR 83 /* 14 bits of eff. address. */ -#define R_PARISC_DIR14DR 84 /* 14 bits of eff. address. */ -#define R_PARISC_DIR16F 85 /* 16 bits of eff. address. */ -#define R_PARISC_DIR16WF 86 /* 16 bits of eff. address. */ -#define R_PARISC_DIR16DF 87 /* 16 bits of eff. address. */ -#define R_PARISC_GPREL64 88 /* 64 bits of GP-rel. address. */ -#define R_PARISC_GPREL14WR 91 /* GP-rel. address, right 14 bits. */ -#define R_PARISC_GPREL14DR 92 /* GP-rel. address, right 14 bits. */ -#define R_PARISC_GPREL16F 93 /* 16 bits GP-rel. address. */ -#define R_PARISC_GPREL16WF 94 /* 16 bits GP-rel. address. */ -#define R_PARISC_GPREL16DF 95 /* 16 bits GP-rel. address. */ -#define R_PARISC_LTOFF64 96 /* 64 bits LT-rel. address. */ -#define R_PARISC_LTOFF14WR 99 /* LT-rel. address, right 14 bits. */ -#define R_PARISC_LTOFF14DR 100 /* LT-rel. address, right 14 bits. */ -#define R_PARISC_LTOFF16F 101 /* 16 bits LT-rel. address. */ -#define R_PARISC_LTOFF16WF 102 /* 16 bits LT-rel. address. */ -#define R_PARISC_LTOFF16DF 103 /* 16 bits LT-rel. address. */ -#define R_PARISC_SECREL64 104 /* 64 bits section rel. address. */ -#define R_PARISC_SEGREL64 112 /* 64 bits segment rel. address. */ -#define R_PARISC_PLTOFF14WR 115 /* PLT-rel. address, right 14 bits. */ -#define R_PARISC_PLTOFF14DR 116 /* PLT-rel. address, right 14 bits. */ -#define R_PARISC_PLTOFF16F 117 /* 16 bits LT-rel. address. */ -#define R_PARISC_PLTOFF16WF 118 /* 16 bits PLT-rel. address. */ -#define R_PARISC_PLTOFF16DF 119 /* 16 bits PLT-rel. address. */ -#define R_PARISC_LTOFF_FPTR64 120 /* 64 bits LT-rel. function ptr. */ -#define R_PARISC_LTOFF_FPTR14WR 123 /* LT-rel. fct. ptr., right 14 bits. */ -#define R_PARISC_LTOFF_FPTR14DR 124 /* LT-rel. fct. ptr., right 14 bits. */ -#define R_PARISC_LTOFF_FPTR16F 125 /* 16 bits LT-rel. function ptr. */ -#define R_PARISC_LTOFF_FPTR16WF 126 /* 16 bits LT-rel. function ptr. */ -#define R_PARISC_LTOFF_FPTR16DF 127 /* 16 bits LT-rel. function ptr. */ -#define R_PARISC_LORESERVE 128 -#define R_PARISC_COPY 128 /* Copy relocation. */ -#define R_PARISC_IPLT 129 /* Dynamic reloc, imported PLT */ -#define R_PARISC_EPLT 130 /* Dynamic reloc, exported PLT */ -#define R_PARISC_TPREL32 153 /* 32 bits TP-rel. address. */ -#define R_PARISC_TPREL21L 154 /* TP-rel. address, left 21 bits. */ -#define R_PARISC_TPREL14R 158 /* TP-rel. address, right 14 bits. */ -#define R_PARISC_LTOFF_TP21L 162 /* LT-TP-rel. address, left 21 bits. */ -#define R_PARISC_LTOFF_TP14R 166 /* LT-TP-rel. address, right 14 bits.*/ -#define R_PARISC_LTOFF_TP14F 167 /* 14 bits LT-TP-rel. address. */ -#define R_PARISC_TPREL64 216 /* 64 bits TP-rel. address. */ -#define R_PARISC_TPREL14WR 219 /* TP-rel. address, right 14 bits. */ -#define R_PARISC_TPREL14DR 220 /* TP-rel. address, right 14 bits. */ -#define R_PARISC_TPREL16F 221 /* 16 bits TP-rel. address. */ -#define R_PARISC_TPREL16WF 222 /* 16 bits TP-rel. address. */ -#define R_PARISC_TPREL16DF 223 /* 16 bits TP-rel. address. */ -#define R_PARISC_LTOFF_TP64 224 /* 64 bits LT-TP-rel. address. */ -#define R_PARISC_LTOFF_TP14WR 227 /* LT-TP-rel. address, right 14 bits.*/ -#define R_PARISC_LTOFF_TP14DR 228 /* LT-TP-rel. address, right 14 bits.*/ -#define R_PARISC_LTOFF_TP16F 229 /* 16 bits LT-TP-rel. address. */ -#define R_PARISC_LTOFF_TP16WF 230 /* 16 bits LT-TP-rel. address. */ -#define R_PARISC_LTOFF_TP16DF 231 /* 16 bits LT-TP-rel. address. */ -#define R_PARISC_GNU_VTENTRY 232 -#define R_PARISC_GNU_VTINHERIT 233 -#define R_PARISC_TLS_GD21L 234 /* GD 21-bit left. */ -#define R_PARISC_TLS_GD14R 235 /* GD 14-bit right. */ -#define R_PARISC_TLS_GDCALL 236 /* GD call to __t_g_a. */ -#define R_PARISC_TLS_LDM21L 237 /* LD module 21-bit left. */ -#define R_PARISC_TLS_LDM14R 238 /* LD module 14-bit right. */ -#define R_PARISC_TLS_LDMCALL 239 /* LD module call to __t_g_a. */ -#define R_PARISC_TLS_LDO21L 240 /* LD offset 21-bit left. */ -#define R_PARISC_TLS_LDO14R 241 /* LD offset 14-bit right. */ -#define R_PARISC_TLS_DTPMOD32 242 /* DTP module 32-bit. */ -#define R_PARISC_TLS_DTPMOD64 243 /* DTP module 64-bit. */ -#define R_PARISC_TLS_DTPOFF32 244 /* DTP offset 32-bit. */ -#define R_PARISC_TLS_DTPOFF64 245 /* DTP offset 32-bit. */ -#define R_PARISC_TLS_LE21L R_PARISC_TPREL21L -#define R_PARISC_TLS_LE14R R_PARISC_TPREL14R -#define R_PARISC_TLS_IE21L R_PARISC_LTOFF_TP21L -#define R_PARISC_TLS_IE14R R_PARISC_LTOFF_TP14R -#define R_PARISC_TLS_TPREL32 R_PARISC_TPREL32 -#define R_PARISC_TLS_TPREL64 R_PARISC_TPREL64 -#define R_PARISC_HIRESERVE 255 - -/* Legal values for p_type field of Elf32_Phdr/Elf64_Phdr. */ - -#define PT_HP_TLS (PT_LOOS + 0x0) -#define PT_HP_CORE_NONE (PT_LOOS + 0x1) -#define PT_HP_CORE_VERSION (PT_LOOS + 0x2) -#define PT_HP_CORE_KERNEL (PT_LOOS + 0x3) -#define PT_HP_CORE_COMM (PT_LOOS + 0x4) -#define PT_HP_CORE_PROC (PT_LOOS + 0x5) -#define PT_HP_CORE_LOADABLE (PT_LOOS + 0x6) -#define PT_HP_CORE_STACK (PT_LOOS + 0x7) -#define PT_HP_CORE_SHM (PT_LOOS + 0x8) -#define PT_HP_CORE_MMF (PT_LOOS + 0x9) -#define PT_HP_PARALLEL (PT_LOOS + 0x10) -#define PT_HP_FASTBIND (PT_LOOS + 0x11) -#define PT_HP_OPT_ANNOT (PT_LOOS + 0x12) -#define PT_HP_HSL_ANNOT (PT_LOOS + 0x13) -#define PT_HP_STACK (PT_LOOS + 0x14) - -#define PT_PARISC_ARCHEXT 0x70000000 -#define PT_PARISC_UNWIND 0x70000001 - -/* Legal values for p_flags field of Elf32_Phdr/Elf64_Phdr. */ - -#define PF_PARISC_SBP 0x08000000 - -#define PF_HP_PAGE_SIZE 0x00100000 -#define PF_HP_FAR_SHARED 0x00200000 -#define PF_HP_NEAR_SHARED 0x00400000 -#define PF_HP_CODE 0x01000000 -#define PF_HP_MODIFY 0x02000000 -#define PF_HP_LAZYSWAP 0x04000000 -#define PF_HP_SBP 0x08000000 - - -/* Alpha specific definitions. */ - -/* Legal values for e_flags field of Elf64_Ehdr. */ - -#define EF_ALPHA_32BIT 1 /* All addresses must be < 2GB. */ -#define EF_ALPHA_CANRELAX 2 /* Relocations for relaxing exist. */ - -/* Legal values for sh_type field of Elf64_Shdr. */ - -/* These two are primerily concerned with ECOFF debugging info. */ -#define SHT_ALPHA_DEBUG 0x70000001 -#define SHT_ALPHA_REGINFO 0x70000002 - -/* Legal values for sh_flags field of Elf64_Shdr. */ - -#define SHF_ALPHA_GPREL 0x10000000 - -/* Legal values for st_other field of Elf64_Sym. */ -#define STO_ALPHA_NOPV 0x80 /* No PV required. */ -#define STO_ALPHA_STD_GPLOAD 0x88 /* PV only used for initial ldgp. */ - -/* Alpha relocs. */ - -#define R_ALPHA_NONE 0 /* No reloc */ -#define R_ALPHA_REFLONG 1 /* Direct 32 bit */ -#define R_ALPHA_REFQUAD 2 /* Direct 64 bit */ -#define R_ALPHA_GPREL32 3 /* GP relative 32 bit */ -#define R_ALPHA_LITERAL 4 /* GP relative 16 bit w/optimization */ -#define R_ALPHA_LITUSE 5 /* Optimization hint for LITERAL */ -#define R_ALPHA_GPDISP 6 /* Add displacement to GP */ -#define R_ALPHA_BRADDR 7 /* PC+4 relative 23 bit shifted */ -#define R_ALPHA_HINT 8 /* PC+4 relative 16 bit shifted */ -#define R_ALPHA_SREL16 9 /* PC relative 16 bit */ -#define R_ALPHA_SREL32 10 /* PC relative 32 bit */ -#define R_ALPHA_SREL64 11 /* PC relative 64 bit */ -#define R_ALPHA_GPRELHIGH 17 /* GP relative 32 bit, high 16 bits */ -#define R_ALPHA_GPRELLOW 18 /* GP relative 32 bit, low 16 bits */ -#define R_ALPHA_GPREL16 19 /* GP relative 16 bit */ -#define R_ALPHA_COPY 24 /* Copy symbol at runtime */ -#define R_ALPHA_GLOB_DAT 25 /* Create GOT entry */ -#define R_ALPHA_JMP_SLOT 26 /* Create PLT entry */ -#define R_ALPHA_RELATIVE 27 /* Adjust by program base */ -#define R_ALPHA_TLS_GD_HI 28 -#define R_ALPHA_TLSGD 29 -#define R_ALPHA_TLS_LDM 30 -#define R_ALPHA_DTPMOD64 31 -#define R_ALPHA_GOTDTPREL 32 -#define R_ALPHA_DTPREL64 33 -#define R_ALPHA_DTPRELHI 34 -#define R_ALPHA_DTPRELLO 35 -#define R_ALPHA_DTPREL16 36 -#define R_ALPHA_GOTTPREL 37 -#define R_ALPHA_TPREL64 38 -#define R_ALPHA_TPRELHI 39 -#define R_ALPHA_TPRELLO 40 -#define R_ALPHA_TPREL16 41 -/* Keep this the last entry. */ -#define R_ALPHA_NUM 46 - -/* Magic values of the LITUSE relocation addend. */ -#define LITUSE_ALPHA_ADDR 0 -#define LITUSE_ALPHA_BASE 1 -#define LITUSE_ALPHA_BYTOFF 2 -#define LITUSE_ALPHA_JSR 3 -#define LITUSE_ALPHA_TLS_GD 4 -#define LITUSE_ALPHA_TLS_LDM 5 - -/* Legal values for d_tag of Elf64_Dyn. */ -#define DT_ALPHA_PLTRO (DT_LOPROC + 0) -#define DT_ALPHA_NUM 1 - -/* PowerPC specific declarations */ - -/* Values for Elf32/64_Ehdr.e_flags. */ -#define EF_PPC_EMB 0x80000000 /* PowerPC embedded flag */ - -/* Cygnus local bits below */ -#define EF_PPC_RELOCATABLE 0x00010000 /* PowerPC -mrelocatable flag*/ -#define EF_PPC_RELOCATABLE_LIB 0x00008000 /* PowerPC -mrelocatable-lib - flag */ - -/* PowerPC relocations defined by the ABIs */ -#define R_PPC_NONE 0 -#define R_PPC_ADDR32 1 /* 32bit absolute address */ -#define R_PPC_ADDR24 2 /* 26bit address, 2 bits ignored. */ -#define R_PPC_ADDR16 3 /* 16bit absolute address */ -#define R_PPC_ADDR16_LO 4 /* lower 16bit of absolute address */ -#define R_PPC_ADDR16_HI 5 /* high 16bit of absolute address */ -#define R_PPC_ADDR16_HA 6 /* adjusted high 16bit */ -#define R_PPC_ADDR14 7 /* 16bit address, 2 bits ignored */ -#define R_PPC_ADDR14_BRTAKEN 8 -#define R_PPC_ADDR14_BRNTAKEN 9 -#define R_PPC_REL24 10 /* PC relative 26 bit */ -#define R_PPC_REL14 11 /* PC relative 16 bit */ -#define R_PPC_REL14_BRTAKEN 12 -#define R_PPC_REL14_BRNTAKEN 13 -#define R_PPC_GOT16 14 -#define R_PPC_GOT16_LO 15 -#define R_PPC_GOT16_HI 16 -#define R_PPC_GOT16_HA 17 -#define R_PPC_PLTREL24 18 -#define R_PPC_COPY 19 -#define R_PPC_GLOB_DAT 20 -#define R_PPC_JMP_SLOT 21 -#define R_PPC_RELATIVE 22 -#define R_PPC_LOCAL24PC 23 -#define R_PPC_UADDR32 24 -#define R_PPC_UADDR16 25 -#define R_PPC_REL32 26 -#define R_PPC_PLT32 27 -#define R_PPC_PLTREL32 28 -#define R_PPC_PLT16_LO 29 -#define R_PPC_PLT16_HI 30 -#define R_PPC_PLT16_HA 31 -#define R_PPC_SDAREL16 32 -#define R_PPC_SECTOFF 33 -#define R_PPC_SECTOFF_LO 34 -#define R_PPC_SECTOFF_HI 35 -#define R_PPC_SECTOFF_HA 36 - -/* PowerPC relocations defined for the TLS access ABI. */ -#define R_PPC_TLS 67 /* none (sym+add)@tls */ -#define R_PPC_DTPMOD32 68 /* word32 (sym+add)@dtpmod */ -#define R_PPC_TPREL16 69 /* half16* (sym+add)@tprel */ -#define R_PPC_TPREL16_LO 70 /* half16 (sym+add)@tprel@l */ -#define R_PPC_TPREL16_HI 71 /* half16 (sym+add)@tprel@h */ -#define R_PPC_TPREL16_HA 72 /* half16 (sym+add)@tprel@ha */ -#define R_PPC_TPREL32 73 /* word32 (sym+add)@tprel */ -#define R_PPC_DTPREL16 74 /* half16* (sym+add)@dtprel */ -#define R_PPC_DTPREL16_LO 75 /* half16 (sym+add)@dtprel@l */ -#define R_PPC_DTPREL16_HI 76 /* half16 (sym+add)@dtprel@h */ -#define R_PPC_DTPREL16_HA 77 /* half16 (sym+add)@dtprel@ha */ -#define R_PPC_DTPREL32 78 /* word32 (sym+add)@dtprel */ -#define R_PPC_GOT_TLSGD16 79 /* half16* (sym+add)@got@tlsgd */ -#define R_PPC_GOT_TLSGD16_LO 80 /* half16 (sym+add)@got@tlsgd@l */ -#define R_PPC_GOT_TLSGD16_HI 81 /* half16 (sym+add)@got@tlsgd@h */ -#define R_PPC_GOT_TLSGD16_HA 82 /* half16 (sym+add)@got@tlsgd@ha */ -#define R_PPC_GOT_TLSLD16 83 /* half16* (sym+add)@got@tlsld */ -#define R_PPC_GOT_TLSLD16_LO 84 /* half16 (sym+add)@got@tlsld@l */ -#define R_PPC_GOT_TLSLD16_HI 85 /* half16 (sym+add)@got@tlsld@h */ -#define R_PPC_GOT_TLSLD16_HA 86 /* half16 (sym+add)@got@tlsld@ha */ -#define R_PPC_GOT_TPREL16 87 /* half16* (sym+add)@got@tprel */ -#define R_PPC_GOT_TPREL16_LO 88 /* half16 (sym+add)@got@tprel@l */ -#define R_PPC_GOT_TPREL16_HI 89 /* half16 (sym+add)@got@tprel@h */ -#define R_PPC_GOT_TPREL16_HA 90 /* half16 (sym+add)@got@tprel@ha */ -#define R_PPC_GOT_DTPREL16 91 /* half16* (sym+add)@got@dtprel */ -#define R_PPC_GOT_DTPREL16_LO 92 /* half16* (sym+add)@got@dtprel@l */ -#define R_PPC_GOT_DTPREL16_HI 93 /* half16* (sym+add)@got@dtprel@h */ -#define R_PPC_GOT_DTPREL16_HA 94 /* half16* (sym+add)@got@dtprel@ha */ - -/* The remaining relocs are from the Embedded ELF ABI, and are not - in the SVR4 ELF ABI. */ -#define R_PPC_EMB_NADDR32 101 -#define R_PPC_EMB_NADDR16 102 -#define R_PPC_EMB_NADDR16_LO 103 -#define R_PPC_EMB_NADDR16_HI 104 -#define R_PPC_EMB_NADDR16_HA 105 -#define R_PPC_EMB_SDAI16 106 -#define R_PPC_EMB_SDA2I16 107 -#define R_PPC_EMB_SDA2REL 108 -#define R_PPC_EMB_SDA21 109 /* 16 bit offset in SDA */ -#define R_PPC_EMB_MRKREF 110 -#define R_PPC_EMB_RELSEC16 111 -#define R_PPC_EMB_RELST_LO 112 -#define R_PPC_EMB_RELST_HI 113 -#define R_PPC_EMB_RELST_HA 114 -#define R_PPC_EMB_BIT_FLD 115 -#define R_PPC_EMB_RELSDA 116 /* 16 bit relative offset in SDA */ - -/* Diab tool relocations. */ -#define R_PPC_DIAB_SDA21_LO 180 /* like EMB_SDA21, but lower 16 bit */ -#define R_PPC_DIAB_SDA21_HI 181 /* like EMB_SDA21, but high 16 bit */ -#define R_PPC_DIAB_SDA21_HA 182 /* like EMB_SDA21, adjusted high 16 */ -#define R_PPC_DIAB_RELSDA_LO 183 /* like EMB_RELSDA, but lower 16 bit */ -#define R_PPC_DIAB_RELSDA_HI 184 /* like EMB_RELSDA, but high 16 bit */ -#define R_PPC_DIAB_RELSDA_HA 185 /* like EMB_RELSDA, adjusted high 16 */ - -/* GNU extension to support local ifunc. */ -#define R_PPC_IRELATIVE 248 - -/* GNU relocs used in PIC code sequences. */ -#define R_PPC_REL16 249 /* half16 (sym+add-.) */ -#define R_PPC_REL16_LO 250 /* half16 (sym+add-.)@l */ -#define R_PPC_REL16_HI 251 /* half16 (sym+add-.)@h */ -#define R_PPC_REL16_HA 252 /* half16 (sym+add-.)@ha */ - -/* This is a phony reloc to handle any old fashioned TOC16 references - that may still be in object files. */ -#define R_PPC_TOC16 255 - -/* PowerPC specific values for the Dyn d_tag field. */ -#define DT_PPC_GOT (DT_LOPROC + 0) -#define DT_PPC_NUM 1 - -/* PowerPC64 relocations defined by the ABIs */ -#define R_PPC64_NONE R_PPC_NONE -#define R_PPC64_ADDR32 R_PPC_ADDR32 /* 32bit absolute address */ -#define R_PPC64_ADDR24 R_PPC_ADDR24 /* 26bit address, word aligned */ -#define R_PPC64_ADDR16 R_PPC_ADDR16 /* 16bit absolute address */ -#define R_PPC64_ADDR16_LO R_PPC_ADDR16_LO /* lower 16bits of address */ -#define R_PPC64_ADDR16_HI R_PPC_ADDR16_HI /* high 16bits of address. */ -#define R_PPC64_ADDR16_HA R_PPC_ADDR16_HA /* adjusted high 16bits. */ -#define R_PPC64_ADDR14 R_PPC_ADDR14 /* 16bit address, word aligned */ -#define R_PPC64_ADDR14_BRTAKEN R_PPC_ADDR14_BRTAKEN -#define R_PPC64_ADDR14_BRNTAKEN R_PPC_ADDR14_BRNTAKEN -#define R_PPC64_REL24 R_PPC_REL24 /* PC-rel. 26 bit, word aligned */ -#define R_PPC64_REL14 R_PPC_REL14 /* PC relative 16 bit */ -#define R_PPC64_REL14_BRTAKEN R_PPC_REL14_BRTAKEN -#define R_PPC64_REL14_BRNTAKEN R_PPC_REL14_BRNTAKEN -#define R_PPC64_GOT16 R_PPC_GOT16 -#define R_PPC64_GOT16_LO R_PPC_GOT16_LO -#define R_PPC64_GOT16_HI R_PPC_GOT16_HI -#define R_PPC64_GOT16_HA R_PPC_GOT16_HA - -#define R_PPC64_COPY R_PPC_COPY -#define R_PPC64_GLOB_DAT R_PPC_GLOB_DAT -#define R_PPC64_JMP_SLOT R_PPC_JMP_SLOT -#define R_PPC64_RELATIVE R_PPC_RELATIVE - -#define R_PPC64_UADDR32 R_PPC_UADDR32 -#define R_PPC64_UADDR16 R_PPC_UADDR16 -#define R_PPC64_REL32 R_PPC_REL32 -#define R_PPC64_PLT32 R_PPC_PLT32 -#define R_PPC64_PLTREL32 R_PPC_PLTREL32 -#define R_PPC64_PLT16_LO R_PPC_PLT16_LO -#define R_PPC64_PLT16_HI R_PPC_PLT16_HI -#define R_PPC64_PLT16_HA R_PPC_PLT16_HA - -#define R_PPC64_SECTOFF R_PPC_SECTOFF -#define R_PPC64_SECTOFF_LO R_PPC_SECTOFF_LO -#define R_PPC64_SECTOFF_HI R_PPC_SECTOFF_HI -#define R_PPC64_SECTOFF_HA R_PPC_SECTOFF_HA -#define R_PPC64_ADDR30 37 /* word30 (S + A - P) >> 2 */ -#define R_PPC64_ADDR64 38 /* doubleword64 S + A */ -#define R_PPC64_ADDR16_HIGHER 39 /* half16 #higher(S + A) */ -#define R_PPC64_ADDR16_HIGHERA 40 /* half16 #highera(S + A) */ -#define R_PPC64_ADDR16_HIGHEST 41 /* half16 #highest(S + A) */ -#define R_PPC64_ADDR16_HIGHESTA 42 /* half16 #highesta(S + A) */ -#define R_PPC64_UADDR64 43 /* doubleword64 S + A */ -#define R_PPC64_REL64 44 /* doubleword64 S + A - P */ -#define R_PPC64_PLT64 45 /* doubleword64 L + A */ -#define R_PPC64_PLTREL64 46 /* doubleword64 L + A - P */ -#define R_PPC64_TOC16 47 /* half16* S + A - .TOC */ -#define R_PPC64_TOC16_LO 48 /* half16 #lo(S + A - .TOC.) */ -#define R_PPC64_TOC16_HI 49 /* half16 #hi(S + A - .TOC.) */ -#define R_PPC64_TOC16_HA 50 /* half16 #ha(S + A - .TOC.) */ -#define R_PPC64_TOC 51 /* doubleword64 .TOC */ -#define R_PPC64_PLTGOT16 52 /* half16* M + A */ -#define R_PPC64_PLTGOT16_LO 53 /* half16 #lo(M + A) */ -#define R_PPC64_PLTGOT16_HI 54 /* half16 #hi(M + A) */ -#define R_PPC64_PLTGOT16_HA 55 /* half16 #ha(M + A) */ - -#define R_PPC64_ADDR16_DS 56 /* half16ds* (S + A) >> 2 */ -#define R_PPC64_ADDR16_LO_DS 57 /* half16ds #lo(S + A) >> 2 */ -#define R_PPC64_GOT16_DS 58 /* half16ds* (G + A) >> 2 */ -#define R_PPC64_GOT16_LO_DS 59 /* half16ds #lo(G + A) >> 2 */ -#define R_PPC64_PLT16_LO_DS 60 /* half16ds #lo(L + A) >> 2 */ -#define R_PPC64_SECTOFF_DS 61 /* half16ds* (R + A) >> 2 */ -#define R_PPC64_SECTOFF_LO_DS 62 /* half16ds #lo(R + A) >> 2 */ -#define R_PPC64_TOC16_DS 63 /* half16ds* (S + A - .TOC.) >> 2 */ -#define R_PPC64_TOC16_LO_DS 64 /* half16ds #lo(S + A - .TOC.) >> 2 */ -#define R_PPC64_PLTGOT16_DS 65 /* half16ds* (M + A) >> 2 */ -#define R_PPC64_PLTGOT16_LO_DS 66 /* half16ds #lo(M + A) >> 2 */ - -/* PowerPC64 relocations defined for the TLS access ABI. */ -#define R_PPC64_TLS 67 /* none (sym+add)@tls */ -#define R_PPC64_DTPMOD64 68 /* doubleword64 (sym+add)@dtpmod */ -#define R_PPC64_TPREL16 69 /* half16* (sym+add)@tprel */ -#define R_PPC64_TPREL16_LO 70 /* half16 (sym+add)@tprel@l */ -#define R_PPC64_TPREL16_HI 71 /* half16 (sym+add)@tprel@h */ -#define R_PPC64_TPREL16_HA 72 /* half16 (sym+add)@tprel@ha */ -#define R_PPC64_TPREL64 73 /* doubleword64 (sym+add)@tprel */ -#define R_PPC64_DTPREL16 74 /* half16* (sym+add)@dtprel */ -#define R_PPC64_DTPREL16_LO 75 /* half16 (sym+add)@dtprel@l */ -#define R_PPC64_DTPREL16_HI 76 /* half16 (sym+add)@dtprel@h */ -#define R_PPC64_DTPREL16_HA 77 /* half16 (sym+add)@dtprel@ha */ -#define R_PPC64_DTPREL64 78 /* doubleword64 (sym+add)@dtprel */ -#define R_PPC64_GOT_TLSGD16 79 /* half16* (sym+add)@got@tlsgd */ -#define R_PPC64_GOT_TLSGD16_LO 80 /* half16 (sym+add)@got@tlsgd@l */ -#define R_PPC64_GOT_TLSGD16_HI 81 /* half16 (sym+add)@got@tlsgd@h */ -#define R_PPC64_GOT_TLSGD16_HA 82 /* half16 (sym+add)@got@tlsgd@ha */ -#define R_PPC64_GOT_TLSLD16 83 /* half16* (sym+add)@got@tlsld */ -#define R_PPC64_GOT_TLSLD16_LO 84 /* half16 (sym+add)@got@tlsld@l */ -#define R_PPC64_GOT_TLSLD16_HI 85 /* half16 (sym+add)@got@tlsld@h */ -#define R_PPC64_GOT_TLSLD16_HA 86 /* half16 (sym+add)@got@tlsld@ha */ -#define R_PPC64_GOT_TPREL16_DS 87 /* half16ds* (sym+add)@got@tprel */ -#define R_PPC64_GOT_TPREL16_LO_DS 88 /* half16ds (sym+add)@got@tprel@l */ -#define R_PPC64_GOT_TPREL16_HI 89 /* half16 (sym+add)@got@tprel@h */ -#define R_PPC64_GOT_TPREL16_HA 90 /* half16 (sym+add)@got@tprel@ha */ -#define R_PPC64_GOT_DTPREL16_DS 91 /* half16ds* (sym+add)@got@dtprel */ -#define R_PPC64_GOT_DTPREL16_LO_DS 92 /* half16ds (sym+add)@got@dtprel@l */ -#define R_PPC64_GOT_DTPREL16_HI 93 /* half16 (sym+add)@got@dtprel@h */ -#define R_PPC64_GOT_DTPREL16_HA 94 /* half16 (sym+add)@got@dtprel@ha */ -#define R_PPC64_TPREL16_DS 95 /* half16ds* (sym+add)@tprel */ -#define R_PPC64_TPREL16_LO_DS 96 /* half16ds (sym+add)@tprel@l */ -#define R_PPC64_TPREL16_HIGHER 97 /* half16 (sym+add)@tprel@higher */ -#define R_PPC64_TPREL16_HIGHERA 98 /* half16 (sym+add)@tprel@highera */ -#define R_PPC64_TPREL16_HIGHEST 99 /* half16 (sym+add)@tprel@highest */ -#define R_PPC64_TPREL16_HIGHESTA 100 /* half16 (sym+add)@tprel@highesta */ -#define R_PPC64_DTPREL16_DS 101 /* half16ds* (sym+add)@dtprel */ -#define R_PPC64_DTPREL16_LO_DS 102 /* half16ds (sym+add)@dtprel@l */ -#define R_PPC64_DTPREL16_HIGHER 103 /* half16 (sym+add)@dtprel@higher */ -#define R_PPC64_DTPREL16_HIGHERA 104 /* half16 (sym+add)@dtprel@highera */ -#define R_PPC64_DTPREL16_HIGHEST 105 /* half16 (sym+add)@dtprel@highest */ -#define R_PPC64_DTPREL16_HIGHESTA 106 /* half16 (sym+add)@dtprel@highesta */ - -/* GNU extension to support local ifunc. */ -#define R_PPC64_JMP_IREL 247 -#define R_PPC64_IRELATIVE 248 -#define R_PPC64_REL16 249 /* half16 (sym+add-.) */ -#define R_PPC64_REL16_LO 250 /* half16 (sym+add-.)@l */ -#define R_PPC64_REL16_HI 251 /* half16 (sym+add-.)@h */ -#define R_PPC64_REL16_HA 252 /* half16 (sym+add-.)@ha */ - -/* PowerPC64 specific values for the Dyn d_tag field. */ -#define DT_PPC64_GLINK (DT_LOPROC + 0) -#define DT_PPC64_OPD (DT_LOPROC + 1) -#define DT_PPC64_OPDSZ (DT_LOPROC + 2) -#define DT_PPC64_NUM 3 - - -/* ARM specific declarations */ - -/* Processor specific flags for the ELF header e_flags field. */ -#define EF_ARM_RELEXEC 0x01 -#define EF_ARM_HASENTRY 0x02 -#define EF_ARM_INTERWORK 0x04 -#define EF_ARM_APCS_26 0x08 -#define EF_ARM_APCS_FLOAT 0x10 -#define EF_ARM_PIC 0x20 -#define EF_ARM_ALIGN8 0x40 /* 8-bit structure alignment is in use */ -#define EF_ARM_NEW_ABI 0x80 -#define EF_ARM_OLD_ABI 0x100 -#define EF_ARM_SOFT_FLOAT 0x200 -#define EF_ARM_VFP_FLOAT 0x400 -#define EF_ARM_MAVERICK_FLOAT 0x800 - - -/* Other constants defined in the ARM ELF spec. version B-01. */ -/* NB. These conflict with values defined above. */ -#define EF_ARM_SYMSARESORTED 0x04 -#define EF_ARM_DYNSYMSUSESEGIDX 0x08 -#define EF_ARM_MAPSYMSFIRST 0x10 -#define EF_ARM_EABIMASK 0XFF000000 - -/* Constants defined in AAELF. */ -#define EF_ARM_BE8 0x00800000 -#define EF_ARM_LE8 0x00400000 - -#define EF_ARM_EABI_VERSION(flags) ((flags) & EF_ARM_EABIMASK) -#define EF_ARM_EABI_UNKNOWN 0x00000000 -#define EF_ARM_EABI_VER1 0x01000000 -#define EF_ARM_EABI_VER2 0x02000000 -#define EF_ARM_EABI_VER3 0x03000000 -#define EF_ARM_EABI_VER4 0x04000000 -#define EF_ARM_EABI_VER5 0x05000000 - -/* Additional symbol types for Thumb. */ -#define STT_ARM_TFUNC STT_LOPROC /* A Thumb function. */ -#define STT_ARM_16BIT STT_HIPROC /* A Thumb label. */ - -/* ARM-specific values for sh_flags */ -#define SHF_ARM_ENTRYSECT 0x10000000 /* Section contains an entry point */ -#define SHF_ARM_COMDEF 0x80000000 /* Section may be multiply defined - in the input to a link step. */ - -/* ARM-specific program header flags */ -#define PF_ARM_SB 0x10000000 /* Segment contains the location - addressed by the static base. */ -#define PF_ARM_PI 0x20000000 /* Position-independent segment. */ -#define PF_ARM_ABS 0x40000000 /* Absolute segment. */ - -/* Processor specific values for the Phdr p_type field. */ -#define PT_ARM_EXIDX (PT_LOPROC + 1) /* ARM unwind segment. */ - -/* Processor specific values for the Shdr sh_type field. */ -#define SHT_ARM_EXIDX (SHT_LOPROC + 1) /* ARM unwind section. */ -#define SHT_ARM_PREEMPTMAP (SHT_LOPROC + 2) /* Preemption details. */ -#define SHT_ARM_ATTRIBUTES (SHT_LOPROC + 3) /* ARM attributes section. */ - - -/* ARM relocs. */ - -#define R_ARM_NONE 0 /* No reloc */ -#define R_ARM_PC24 1 /* PC relative 26 bit branch */ -#define R_ARM_ABS32 2 /* Direct 32 bit */ -#define R_ARM_REL32 3 /* PC relative 32 bit */ -#define R_ARM_PC13 4 -#define R_ARM_ABS16 5 /* Direct 16 bit */ -#define R_ARM_ABS12 6 /* Direct 12 bit */ -#define R_ARM_THM_ABS5 7 -#define R_ARM_ABS8 8 /* Direct 8 bit */ -#define R_ARM_SBREL32 9 -#define R_ARM_THM_PC22 10 -#define R_ARM_THM_PC8 11 -#define R_ARM_AMP_VCALL9 12 -#define R_ARM_SWI24 13 /* Obsolete static relocation. */ -#define R_ARM_TLS_DESC 13 /* Dynamic relocation. */ -#define R_ARM_THM_SWI8 14 -#define R_ARM_XPC25 15 -#define R_ARM_THM_XPC22 16 -#define R_ARM_TLS_DTPMOD32 17 /* ID of module containing symbol */ -#define R_ARM_TLS_DTPOFF32 18 /* Offset in TLS block */ -#define R_ARM_TLS_TPOFF32 19 /* Offset in static TLS block */ -#define R_ARM_COPY 20 /* Copy symbol at runtime */ -#define R_ARM_GLOB_DAT 21 /* Create GOT entry */ -#define R_ARM_JUMP_SLOT 22 /* Create PLT entry */ -#define R_ARM_RELATIVE 23 /* Adjust by program base */ -#define R_ARM_GOTOFF 24 /* 32 bit offset to GOT */ -#define R_ARM_GOTPC 25 /* 32 bit PC relative offset to GOT */ -#define R_ARM_GOT32 26 /* 32 bit GOT entry */ -#define R_ARM_PLT32 27 /* 32 bit PLT address */ -#define R_ARM_ALU_PCREL_7_0 32 -#define R_ARM_ALU_PCREL_15_8 33 -#define R_ARM_ALU_PCREL_23_15 34 -#define R_ARM_LDR_SBREL_11_0 35 -#define R_ARM_ALU_SBREL_19_12 36 -#define R_ARM_ALU_SBREL_27_20 37 -#define R_ARM_TLS_GOTDESC 90 -#define R_ARM_TLS_CALL 91 -#define R_ARM_TLS_DESCSEQ 92 -#define R_ARM_THM_TLS_CALL 93 -#define R_ARM_GNU_VTENTRY 100 -#define R_ARM_GNU_VTINHERIT 101 -#define R_ARM_THM_PC11 102 /* thumb unconditional branch */ -#define R_ARM_THM_PC9 103 /* thumb conditional branch */ -#define R_ARM_TLS_GD32 104 /* PC-rel 32 bit for global dynamic - thread local data */ -#define R_ARM_TLS_LDM32 105 /* PC-rel 32 bit for local dynamic - thread local data */ -#define R_ARM_TLS_LDO32 106 /* 32 bit offset relative to TLS - block */ -#define R_ARM_TLS_IE32 107 /* PC-rel 32 bit for GOT entry of - static TLS block offset */ -#define R_ARM_TLS_LE32 108 /* 32 bit offset relative to static - TLS block */ -#define R_ARM_THM_TLS_DESCSEQ 129 -#define R_ARM_IRELATIVE 160 -#define R_ARM_RXPC25 249 -#define R_ARM_RSBREL32 250 -#define R_ARM_THM_RPC22 251 -#define R_ARM_RREL32 252 -#define R_ARM_RABS22 253 -#define R_ARM_RPC24 254 -#define R_ARM_RBASE 255 -/* Keep this the last entry. */ -#define R_ARM_NUM 256 - -/* IA-64 specific declarations. */ - -/* Processor specific flags for the Ehdr e_flags field. */ -#define EF_IA_64_MASKOS 0x0000000f /* os-specific flags */ -#define EF_IA_64_ABI64 0x00000010 /* 64-bit ABI */ -#define EF_IA_64_ARCH 0xff000000 /* arch. version mask */ - -/* Processor specific values for the Phdr p_type field. */ -#define PT_IA_64_ARCHEXT (PT_LOPROC + 0) /* arch extension bits */ -#define PT_IA_64_UNWIND (PT_LOPROC + 1) /* ia64 unwind bits */ -#define PT_IA_64_HP_OPT_ANOT (PT_LOOS + 0x12) -#define PT_IA_64_HP_HSL_ANOT (PT_LOOS + 0x13) -#define PT_IA_64_HP_STACK (PT_LOOS + 0x14) - -/* Processor specific flags for the Phdr p_flags field. */ -#define PF_IA_64_NORECOV 0x80000000 /* spec insns w/o recovery */ - -/* Processor specific values for the Shdr sh_type field. */ -#define SHT_IA_64_EXT (SHT_LOPROC + 0) /* extension bits */ -#define SHT_IA_64_UNWIND (SHT_LOPROC + 1) /* unwind bits */ - -/* Processor specific flags for the Shdr sh_flags field. */ -#define SHF_IA_64_SHORT 0x10000000 /* section near gp */ -#define SHF_IA_64_NORECOV 0x20000000 /* spec insns w/o recovery */ - -/* Processor specific values for the Dyn d_tag field. */ -#define DT_IA_64_PLT_RESERVE (DT_LOPROC + 0) -#define DT_IA_64_NUM 1 - -/* IA-64 relocations. */ -#define R_IA64_NONE 0x00 /* none */ -#define R_IA64_IMM14 0x21 /* symbol + addend, add imm14 */ -#define R_IA64_IMM22 0x22 /* symbol + addend, add imm22 */ -#define R_IA64_IMM64 0x23 /* symbol + addend, mov imm64 */ -#define R_IA64_DIR32MSB 0x24 /* symbol + addend, data4 MSB */ -#define R_IA64_DIR32LSB 0x25 /* symbol + addend, data4 LSB */ -#define R_IA64_DIR64MSB 0x26 /* symbol + addend, data8 MSB */ -#define R_IA64_DIR64LSB 0x27 /* symbol + addend, data8 LSB */ -#define R_IA64_GPREL22 0x2a /* @gprel(sym + add), add imm22 */ -#define R_IA64_GPREL64I 0x2b /* @gprel(sym + add), mov imm64 */ -#define R_IA64_GPREL32MSB 0x2c /* @gprel(sym + add), data4 MSB */ -#define R_IA64_GPREL32LSB 0x2d /* @gprel(sym + add), data4 LSB */ -#define R_IA64_GPREL64MSB 0x2e /* @gprel(sym + add), data8 MSB */ -#define R_IA64_GPREL64LSB 0x2f /* @gprel(sym + add), data8 LSB */ -#define R_IA64_LTOFF22 0x32 /* @ltoff(sym + add), add imm22 */ -#define R_IA64_LTOFF64I 0x33 /* @ltoff(sym + add), mov imm64 */ -#define R_IA64_PLTOFF22 0x3a /* @pltoff(sym + add), add imm22 */ -#define R_IA64_PLTOFF64I 0x3b /* @pltoff(sym + add), mov imm64 */ -#define R_IA64_PLTOFF64MSB 0x3e /* @pltoff(sym + add), data8 MSB */ -#define R_IA64_PLTOFF64LSB 0x3f /* @pltoff(sym + add), data8 LSB */ -#define R_IA64_FPTR64I 0x43 /* @fptr(sym + add), mov imm64 */ -#define R_IA64_FPTR32MSB 0x44 /* @fptr(sym + add), data4 MSB */ -#define R_IA64_FPTR32LSB 0x45 /* @fptr(sym + add), data4 LSB */ -#define R_IA64_FPTR64MSB 0x46 /* @fptr(sym + add), data8 MSB */ -#define R_IA64_FPTR64LSB 0x47 /* @fptr(sym + add), data8 LSB */ -#define R_IA64_PCREL60B 0x48 /* @pcrel(sym + add), brl */ -#define R_IA64_PCREL21B 0x49 /* @pcrel(sym + add), ptb, call */ -#define R_IA64_PCREL21M 0x4a /* @pcrel(sym + add), chk.s */ -#define R_IA64_PCREL21F 0x4b /* @pcrel(sym + add), fchkf */ -#define R_IA64_PCREL32MSB 0x4c /* @pcrel(sym + add), data4 MSB */ -#define R_IA64_PCREL32LSB 0x4d /* @pcrel(sym + add), data4 LSB */ -#define R_IA64_PCREL64MSB 0x4e /* @pcrel(sym + add), data8 MSB */ -#define R_IA64_PCREL64LSB 0x4f /* @pcrel(sym + add), data8 LSB */ -#define R_IA64_LTOFF_FPTR22 0x52 /* @ltoff(@fptr(s+a)), imm22 */ -#define R_IA64_LTOFF_FPTR64I 0x53 /* @ltoff(@fptr(s+a)), imm64 */ -#define R_IA64_LTOFF_FPTR32MSB 0x54 /* @ltoff(@fptr(s+a)), data4 MSB */ -#define R_IA64_LTOFF_FPTR32LSB 0x55 /* @ltoff(@fptr(s+a)), data4 LSB */ -#define R_IA64_LTOFF_FPTR64MSB 0x56 /* @ltoff(@fptr(s+a)), data8 MSB */ -#define R_IA64_LTOFF_FPTR64LSB 0x57 /* @ltoff(@fptr(s+a)), data8 LSB */ -#define R_IA64_SEGREL32MSB 0x5c /* @segrel(sym + add), data4 MSB */ -#define R_IA64_SEGREL32LSB 0x5d /* @segrel(sym + add), data4 LSB */ -#define R_IA64_SEGREL64MSB 0x5e /* @segrel(sym + add), data8 MSB */ -#define R_IA64_SEGREL64LSB 0x5f /* @segrel(sym + add), data8 LSB */ -#define R_IA64_SECREL32MSB 0x64 /* @secrel(sym + add), data4 MSB */ -#define R_IA64_SECREL32LSB 0x65 /* @secrel(sym + add), data4 LSB */ -#define R_IA64_SECREL64MSB 0x66 /* @secrel(sym + add), data8 MSB */ -#define R_IA64_SECREL64LSB 0x67 /* @secrel(sym + add), data8 LSB */ -#define R_IA64_REL32MSB 0x6c /* data 4 + REL */ -#define R_IA64_REL32LSB 0x6d /* data 4 + REL */ -#define R_IA64_REL64MSB 0x6e /* data 8 + REL */ -#define R_IA64_REL64LSB 0x6f /* data 8 + REL */ -#define R_IA64_LTV32MSB 0x74 /* symbol + addend, data4 MSB */ -#define R_IA64_LTV32LSB 0x75 /* symbol + addend, data4 LSB */ -#define R_IA64_LTV64MSB 0x76 /* symbol + addend, data8 MSB */ -#define R_IA64_LTV64LSB 0x77 /* symbol + addend, data8 LSB */ -#define R_IA64_PCREL21BI 0x79 /* @pcrel(sym + add), 21bit inst */ -#define R_IA64_PCREL22 0x7a /* @pcrel(sym + add), 22bit inst */ -#define R_IA64_PCREL64I 0x7b /* @pcrel(sym + add), 64bit inst */ -#define R_IA64_IPLTMSB 0x80 /* dynamic reloc, imported PLT, MSB */ -#define R_IA64_IPLTLSB 0x81 /* dynamic reloc, imported PLT, LSB */ -#define R_IA64_COPY 0x84 /* copy relocation */ -#define R_IA64_SUB 0x85 /* Addend and symbol difference */ -#define R_IA64_LTOFF22X 0x86 /* LTOFF22, relaxable. */ -#define R_IA64_LDXMOV 0x87 /* Use of LTOFF22X. */ -#define R_IA64_TPREL14 0x91 /* @tprel(sym + add), imm14 */ -#define R_IA64_TPREL22 0x92 /* @tprel(sym + add), imm22 */ -#define R_IA64_TPREL64I 0x93 /* @tprel(sym + add), imm64 */ -#define R_IA64_TPREL64MSB 0x96 /* @tprel(sym + add), data8 MSB */ -#define R_IA64_TPREL64LSB 0x97 /* @tprel(sym + add), data8 LSB */ -#define R_IA64_LTOFF_TPREL22 0x9a /* @ltoff(@tprel(s+a)), imm2 */ -#define R_IA64_DTPMOD64MSB 0xa6 /* @dtpmod(sym + add), data8 MSB */ -#define R_IA64_DTPMOD64LSB 0xa7 /* @dtpmod(sym + add), data8 LSB */ -#define R_IA64_LTOFF_DTPMOD22 0xaa /* @ltoff(@dtpmod(sym + add)), imm22 */ -#define R_IA64_DTPREL14 0xb1 /* @dtprel(sym + add), imm14 */ -#define R_IA64_DTPREL22 0xb2 /* @dtprel(sym + add), imm22 */ -#define R_IA64_DTPREL64I 0xb3 /* @dtprel(sym + add), imm64 */ -#define R_IA64_DTPREL32MSB 0xb4 /* @dtprel(sym + add), data4 MSB */ -#define R_IA64_DTPREL32LSB 0xb5 /* @dtprel(sym + add), data4 LSB */ -#define R_IA64_DTPREL64MSB 0xb6 /* @dtprel(sym + add), data8 MSB */ -#define R_IA64_DTPREL64LSB 0xb7 /* @dtprel(sym + add), data8 LSB */ -#define R_IA64_LTOFF_DTPREL22 0xba /* @ltoff(@dtprel(s+a)), imm22 */ - -/* SH specific declarations */ - -/* Processor specific flags for the ELF header e_flags field. */ -#define EF_SH_MACH_MASK 0x1f -#define EF_SH_UNKNOWN 0x0 -#define EF_SH1 0x1 -#define EF_SH2 0x2 -#define EF_SH3 0x3 -#define EF_SH_DSP 0x4 -#define EF_SH3_DSP 0x5 -#define EF_SH4AL_DSP 0x6 -#define EF_SH3E 0x8 -#define EF_SH4 0x9 -#define EF_SH2E 0xb -#define EF_SH4A 0xc -#define EF_SH2A 0xd -#define EF_SH4_NOFPU 0x10 -#define EF_SH4A_NOFPU 0x11 -#define EF_SH4_NOMMU_NOFPU 0x12 -#define EF_SH2A_NOFPU 0x13 -#define EF_SH3_NOMMU 0x14 -#define EF_SH2A_SH4_NOFPU 0x15 -#define EF_SH2A_SH3_NOFPU 0x16 -#define EF_SH2A_SH4 0x17 -#define EF_SH2A_SH3E 0x18 - -/* SH relocs. */ -#define R_SH_NONE 0 -#define R_SH_DIR32 1 -#define R_SH_REL32 2 -#define R_SH_DIR8WPN 3 -#define R_SH_IND12W 4 -#define R_SH_DIR8WPL 5 -#define R_SH_DIR8WPZ 6 -#define R_SH_DIR8BP 7 -#define R_SH_DIR8W 8 -#define R_SH_DIR8L 9 -#define R_SH_SWITCH16 25 -#define R_SH_SWITCH32 26 -#define R_SH_USES 27 -#define R_SH_COUNT 28 -#define R_SH_ALIGN 29 -#define R_SH_CODE 30 -#define R_SH_DATA 31 -#define R_SH_LABEL 32 -#define R_SH_SWITCH8 33 -#define R_SH_GNU_VTINHERIT 34 -#define R_SH_GNU_VTENTRY 35 -#define R_SH_TLS_GD_32 144 -#define R_SH_TLS_LD_32 145 -#define R_SH_TLS_LDO_32 146 -#define R_SH_TLS_IE_32 147 -#define R_SH_TLS_LE_32 148 -#define R_SH_TLS_DTPMOD32 149 -#define R_SH_TLS_DTPOFF32 150 -#define R_SH_TLS_TPOFF32 151 -#define R_SH_GOT32 160 -#define R_SH_PLT32 161 -#define R_SH_COPY 162 -#define R_SH_GLOB_DAT 163 -#define R_SH_JMP_SLOT 164 -#define R_SH_RELATIVE 165 -#define R_SH_GOTOFF 166 -#define R_SH_GOTPC 167 -/* Keep this the last entry. */ -#define R_SH_NUM 256 - -/* S/390 specific definitions. */ - -/* Valid values for the e_flags field. */ - -#define EF_S390_HIGH_GPRS 0x00000001 /* High GPRs kernel facility needed. */ - -/* Additional s390 relocs */ - -#define R_390_NONE 0 /* No reloc. */ -#define R_390_8 1 /* Direct 8 bit. */ -#define R_390_12 2 /* Direct 12 bit. */ -#define R_390_16 3 /* Direct 16 bit. */ -#define R_390_32 4 /* Direct 32 bit. */ -#define R_390_PC32 5 /* PC relative 32 bit. */ -#define R_390_GOT12 6 /* 12 bit GOT offset. */ -#define R_390_GOT32 7 /* 32 bit GOT offset. */ -#define R_390_PLT32 8 /* 32 bit PC relative PLT address. */ -#define R_390_COPY 9 /* Copy symbol at runtime. */ -#define R_390_GLOB_DAT 10 /* Create GOT entry. */ -#define R_390_JMP_SLOT 11 /* Create PLT entry. */ -#define R_390_RELATIVE 12 /* Adjust by program base. */ -#define R_390_GOTOFF32 13 /* 32 bit offset to GOT. */ -#define R_390_GOTPC 14 /* 32 bit PC relative offset to GOT. */ -#define R_390_GOT16 15 /* 16 bit GOT offset. */ -#define R_390_PC16 16 /* PC relative 16 bit. */ -#define R_390_PC16DBL 17 /* PC relative 16 bit shifted by 1. */ -#define R_390_PLT16DBL 18 /* 16 bit PC rel. PLT shifted by 1. */ -#define R_390_PC32DBL 19 /* PC relative 32 bit shifted by 1. */ -#define R_390_PLT32DBL 20 /* 32 bit PC rel. PLT shifted by 1. */ -#define R_390_GOTPCDBL 21 /* 32 bit PC rel. GOT shifted by 1. */ -#define R_390_64 22 /* Direct 64 bit. */ -#define R_390_PC64 23 /* PC relative 64 bit. */ -#define R_390_GOT64 24 /* 64 bit GOT offset. */ -#define R_390_PLT64 25 /* 64 bit PC relative PLT address. */ -#define R_390_GOTENT 26 /* 32 bit PC rel. to GOT entry >> 1. */ -#define R_390_GOTOFF16 27 /* 16 bit offset to GOT. */ -#define R_390_GOTOFF64 28 /* 64 bit offset to GOT. */ -#define R_390_GOTPLT12 29 /* 12 bit offset to jump slot. */ -#define R_390_GOTPLT16 30 /* 16 bit offset to jump slot. */ -#define R_390_GOTPLT32 31 /* 32 bit offset to jump slot. */ -#define R_390_GOTPLT64 32 /* 64 bit offset to jump slot. */ -#define R_390_GOTPLTENT 33 /* 32 bit rel. offset to jump slot. */ -#define R_390_PLTOFF16 34 /* 16 bit offset from GOT to PLT. */ -#define R_390_PLTOFF32 35 /* 32 bit offset from GOT to PLT. */ -#define R_390_PLTOFF64 36 /* 16 bit offset from GOT to PLT. */ -#define R_390_TLS_LOAD 37 /* Tag for load insn in TLS code. */ -#define R_390_TLS_GDCALL 38 /* Tag for function call in general - dynamic TLS code. */ -#define R_390_TLS_LDCALL 39 /* Tag for function call in local - dynamic TLS code. */ -#define R_390_TLS_GD32 40 /* Direct 32 bit for general dynamic - thread local data. */ -#define R_390_TLS_GD64 41 /* Direct 64 bit for general dynamic - thread local data. */ -#define R_390_TLS_GOTIE12 42 /* 12 bit GOT offset for static TLS - block offset. */ -#define R_390_TLS_GOTIE32 43 /* 32 bit GOT offset for static TLS - block offset. */ -#define R_390_TLS_GOTIE64 44 /* 64 bit GOT offset for static TLS - block offset. */ -#define R_390_TLS_LDM32 45 /* Direct 32 bit for local dynamic - thread local data in LE code. */ -#define R_390_TLS_LDM64 46 /* Direct 64 bit for local dynamic - thread local data in LE code. */ -#define R_390_TLS_IE32 47 /* 32 bit address of GOT entry for - negated static TLS block offset. */ -#define R_390_TLS_IE64 48 /* 64 bit address of GOT entry for - negated static TLS block offset. */ -#define R_390_TLS_IEENT 49 /* 32 bit rel. offset to GOT entry for - negated static TLS block offset. */ -#define R_390_TLS_LE32 50 /* 32 bit negated offset relative to - static TLS block. */ -#define R_390_TLS_LE64 51 /* 64 bit negated offset relative to - static TLS block. */ -#define R_390_TLS_LDO32 52 /* 32 bit offset relative to TLS - block. */ -#define R_390_TLS_LDO64 53 /* 64 bit offset relative to TLS - block. */ -#define R_390_TLS_DTPMOD 54 /* ID of module containing symbol. */ -#define R_390_TLS_DTPOFF 55 /* Offset in TLS block. */ -#define R_390_TLS_TPOFF 56 /* Negated offset in static TLS - block. */ -#define R_390_20 57 /* Direct 20 bit. */ -#define R_390_GOT20 58 /* 20 bit GOT offset. */ -#define R_390_GOTPLT20 59 /* 20 bit offset to jump slot. */ -#define R_390_TLS_GOTIE20 60 /* 20 bit GOT offset for static TLS - block offset. */ -/* Keep this the last entry. */ -#define R_390_NUM 61 - - -/* CRIS relocations. */ -#define R_CRIS_NONE 0 -#define R_CRIS_8 1 -#define R_CRIS_16 2 -#define R_CRIS_32 3 -#define R_CRIS_8_PCREL 4 -#define R_CRIS_16_PCREL 5 -#define R_CRIS_32_PCREL 6 -#define R_CRIS_GNU_VTINHERIT 7 -#define R_CRIS_GNU_VTENTRY 8 -#define R_CRIS_COPY 9 -#define R_CRIS_GLOB_DAT 10 -#define R_CRIS_JUMP_SLOT 11 -#define R_CRIS_RELATIVE 12 -#define R_CRIS_16_GOT 13 -#define R_CRIS_32_GOT 14 -#define R_CRIS_16_GOTPLT 15 -#define R_CRIS_32_GOTPLT 16 -#define R_CRIS_32_GOTREL 17 -#define R_CRIS_32_PLT_GOTREL 18 -#define R_CRIS_32_PLT_PCREL 19 - -#define R_CRIS_NUM 20 - - -/* AMD x86-64 relocations. */ -#define R_X86_64_NONE 0 /* No reloc */ -#define R_X86_64_64 1 /* Direct 64 bit */ -#define R_X86_64_PC32 2 /* PC relative 32 bit signed */ -#define R_X86_64_GOT32 3 /* 32 bit GOT entry */ -#define R_X86_64_PLT32 4 /* 32 bit PLT address */ -#define R_X86_64_COPY 5 /* Copy symbol at runtime */ -#define R_X86_64_GLOB_DAT 6 /* Create GOT entry */ -#define R_X86_64_JUMP_SLOT 7 /* Create PLT entry */ -#define R_X86_64_RELATIVE 8 /* Adjust by program base */ -#define R_X86_64_GOTPCREL 9 /* 32 bit signed PC relative - offset to GOT */ -#define R_X86_64_32 10 /* Direct 32 bit zero extended */ -#define R_X86_64_32S 11 /* Direct 32 bit sign extended */ -#define R_X86_64_16 12 /* Direct 16 bit zero extended */ -#define R_X86_64_PC16 13 /* 16 bit sign extended pc relative */ -#define R_X86_64_8 14 /* Direct 8 bit sign extended */ -#define R_X86_64_PC8 15 /* 8 bit sign extended pc relative */ -#define R_X86_64_DTPMOD64 16 /* ID of module containing symbol */ -#define R_X86_64_DTPOFF64 17 /* Offset in module's TLS block */ -#define R_X86_64_TPOFF64 18 /* Offset in initial TLS block */ -#define R_X86_64_TLSGD 19 /* 32 bit signed PC relative offset - to two GOT entries for GD symbol */ -#define R_X86_64_TLSLD 20 /* 32 bit signed PC relative offset - to two GOT entries for LD symbol */ -#define R_X86_64_DTPOFF32 21 /* Offset in TLS block */ -#define R_X86_64_GOTTPOFF 22 /* 32 bit signed PC relative offset - to GOT entry for IE symbol */ -#define R_X86_64_TPOFF32 23 /* Offset in initial TLS block */ -#define R_X86_64_PC64 24 /* PC relative 64 bit */ -#define R_X86_64_GOTOFF64 25 /* 64 bit offset to GOT */ -#define R_X86_64_GOTPC32 26 /* 32 bit signed pc relative - offset to GOT */ -#define R_X86_64_GOT64 27 /* 64-bit GOT entry offset */ -#define R_X86_64_GOTPCREL64 28 /* 64-bit PC relative offset - to GOT entry */ -#define R_X86_64_GOTPC64 29 /* 64-bit PC relative offset to GOT */ -#define R_X86_64_GOTPLT64 30 /* like GOT64, says PLT entry needed */ -#define R_X86_64_PLTOFF64 31 /* 64-bit GOT relative offset - to PLT entry */ -#define R_X86_64_SIZE32 32 /* Size of symbol plus 32-bit addend */ -#define R_X86_64_SIZE64 33 /* Size of symbol plus 64-bit addend */ -#define R_X86_64_GOTPC32_TLSDESC 34 /* GOT offset for TLS descriptor. */ -#define R_X86_64_TLSDESC_CALL 35 /* Marker for call through TLS - descriptor. */ -#define R_X86_64_TLSDESC 36 /* TLS descriptor. */ -#define R_X86_64_IRELATIVE 37 /* Adjust indirectly by program base */ - -#define R_X86_64_NUM 38 - - -/* AM33 relocations. */ -#define R_MN10300_NONE 0 /* No reloc. */ -#define R_MN10300_32 1 /* Direct 32 bit. */ -#define R_MN10300_16 2 /* Direct 16 bit. */ -#define R_MN10300_8 3 /* Direct 8 bit. */ -#define R_MN10300_PCREL32 4 /* PC-relative 32-bit. */ -#define R_MN10300_PCREL16 5 /* PC-relative 16-bit signed. */ -#define R_MN10300_PCREL8 6 /* PC-relative 8-bit signed. */ -#define R_MN10300_GNU_VTINHERIT 7 /* Ancient C++ vtable garbage... */ -#define R_MN10300_GNU_VTENTRY 8 /* ... collection annotation. */ -#define R_MN10300_24 9 /* Direct 24 bit. */ -#define R_MN10300_GOTPC32 10 /* 32-bit PCrel offset to GOT. */ -#define R_MN10300_GOTPC16 11 /* 16-bit PCrel offset to GOT. */ -#define R_MN10300_GOTOFF32 12 /* 32-bit offset from GOT. */ -#define R_MN10300_GOTOFF24 13 /* 24-bit offset from GOT. */ -#define R_MN10300_GOTOFF16 14 /* 16-bit offset from GOT. */ -#define R_MN10300_PLT32 15 /* 32-bit PCrel to PLT entry. */ -#define R_MN10300_PLT16 16 /* 16-bit PCrel to PLT entry. */ -#define R_MN10300_GOT32 17 /* 32-bit offset to GOT entry. */ -#define R_MN10300_GOT24 18 /* 24-bit offset to GOT entry. */ -#define R_MN10300_GOT16 19 /* 16-bit offset to GOT entry. */ -#define R_MN10300_COPY 20 /* Copy symbol at runtime. */ -#define R_MN10300_GLOB_DAT 21 /* Create GOT entry. */ -#define R_MN10300_JMP_SLOT 22 /* Create PLT entry. */ -#define R_MN10300_RELATIVE 23 /* Adjust by program base. */ - -#define R_MN10300_NUM 24 - - -/* M32R relocs. */ -#define R_M32R_NONE 0 /* No reloc. */ -#define R_M32R_16 1 /* Direct 16 bit. */ -#define R_M32R_32 2 /* Direct 32 bit. */ -#define R_M32R_24 3 /* Direct 24 bit. */ -#define R_M32R_10_PCREL 4 /* PC relative 10 bit shifted. */ -#define R_M32R_18_PCREL 5 /* PC relative 18 bit shifted. */ -#define R_M32R_26_PCREL 6 /* PC relative 26 bit shifted. */ -#define R_M32R_HI16_ULO 7 /* High 16 bit with unsigned low. */ -#define R_M32R_HI16_SLO 8 /* High 16 bit with signed low. */ -#define R_M32R_LO16 9 /* Low 16 bit. */ -#define R_M32R_SDA16 10 /* 16 bit offset in SDA. */ -#define R_M32R_GNU_VTINHERIT 11 -#define R_M32R_GNU_VTENTRY 12 -/* M32R relocs use SHT_RELA. */ -#define R_M32R_16_RELA 33 /* Direct 16 bit. */ -#define R_M32R_32_RELA 34 /* Direct 32 bit. */ -#define R_M32R_24_RELA 35 /* Direct 24 bit. */ -#define R_M32R_10_PCREL_RELA 36 /* PC relative 10 bit shifted. */ -#define R_M32R_18_PCREL_RELA 37 /* PC relative 18 bit shifted. */ -#define R_M32R_26_PCREL_RELA 38 /* PC relative 26 bit shifted. */ -#define R_M32R_HI16_ULO_RELA 39 /* High 16 bit with unsigned low */ -#define R_M32R_HI16_SLO_RELA 40 /* High 16 bit with signed low */ -#define R_M32R_LO16_RELA 41 /* Low 16 bit */ -#define R_M32R_SDA16_RELA 42 /* 16 bit offset in SDA */ -#define R_M32R_RELA_GNU_VTINHERIT 43 -#define R_M32R_RELA_GNU_VTENTRY 44 -#define R_M32R_REL32 45 /* PC relative 32 bit. */ - -#define R_M32R_GOT24 48 /* 24 bit GOT entry */ -#define R_M32R_26_PLTREL 49 /* 26 bit PC relative to PLT shifted */ -#define R_M32R_COPY 50 /* Copy symbol at runtime */ -#define R_M32R_GLOB_DAT 51 /* Create GOT entry */ -#define R_M32R_JMP_SLOT 52 /* Create PLT entry */ -#define R_M32R_RELATIVE 53 /* Adjust by program base */ -#define R_M32R_GOTOFF 54 /* 24 bit offset to GOT */ -#define R_M32R_GOTPC24 55 /* 24 bit PC relative offset to GOT */ -#define R_M32R_GOT16_HI_ULO 56 /* High 16 bit GOT entry with unsigned - low */ -#define R_M32R_GOT16_HI_SLO 57 /* High 16 bit GOT entry with signed - low */ -#define R_M32R_GOT16_LO 58 /* Low 16 bit GOT entry */ -#define R_M32R_GOTPC_HI_ULO 59 /* High 16 bit PC relative offset to - GOT with unsigned low */ -#define R_M32R_GOTPC_HI_SLO 60 /* High 16 bit PC relative offset to - GOT with signed low */ -#define R_M32R_GOTPC_LO 61 /* Low 16 bit PC relative offset to - GOT */ -#define R_M32R_GOTOFF_HI_ULO 62 /* High 16 bit offset to GOT - with unsigned low */ -#define R_M32R_GOTOFF_HI_SLO 63 /* High 16 bit offset to GOT - with signed low */ -#define R_M32R_GOTOFF_LO 64 /* Low 16 bit offset to GOT */ -#define R_M32R_NUM 256 /* Keep this the last entry. */ - - -__END_DECLS - -#endif /* elf.h */ diff --git a/api/util/tar.hpp b/api/util/tar.hpp index ba897d44a8..97d3f5cfbd 100644 --- a/api/util/tar.hpp +++ b/api/util/tar.hpp @@ -19,7 +19,7 @@ #ifndef TAR_HPP #define TAR_HPP -#include // Our posix header has the Tar_header struct, which the newlib tar.h does not +#include #include // From uzlib (mod) #include #include diff --git a/cmake/cross_compiled_libraries.txt b/cmake/cross_compiled_libraries.txt index dde8a3ab35..3c705cceba 100644 --- a/cmake/cross_compiled_libraries.txt +++ b/cmake/cross_compiled_libraries.txt @@ -80,17 +80,17 @@ add_dependencies(libcxxabi PrecompiledLibraries) set_target_properties(libcxx PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libc++.a) set_target_properties(libcxxabi PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libc++abi.a) -set(NEWLIB_INCLUDE_DIR ${PRECOMPILED_DIR}/newlib/include/) -set(NEWLIB_LIB_DIR ${PRECOMPILED_DIR}/newlib/) +set(MUSL_INCLUDE_DIR ${PRECOMPILED_DIR}/musl/include/) +set(MUSL_LIB_DIR ${PRECOMPILED_DIR}/musl/) set(LIBGCC_LIB_DIR ${PRECOMPILED_DIR}/libgcc/) add_library(libc STATIC IMPORTED) -set_target_properties(libc PROPERTIES IMPORTED_LOCATION ${NEWLIB_LIB_DIR}/libc.a) +set_target_properties(libc PROPERTIES IMPORTED_LOCATION ${MUSL_LIB_DIR}/libc.a) add_dependencies(libc PrecompiledLibraries) add_library(libm STATIC IMPORTED) -set_target_properties(libm PROPERTIES IMPORTED_LOCATION ${NEWLIB_LIB_DIR}/libm.a) +set_target_properties(libm PROPERTIES IMPORTED_LOCATION ${MUSL_LIB_DIR}/libm.a) add_dependencies(libm PrecompiledLibraries) set(CRTEND ${PRECOMPILED_DIR}/crt/crtend.o) @@ -99,8 +99,8 @@ set(CRTBEGIN ${PRECOMPILED_DIR}/crt/crtbegin.o) # installation instructions install(DIRECTORY ${LIBCXX_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/libcxx) -install(DIRECTORY ${NEWLIB_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/newlib) +install(DIRECTORY ${MUSL_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/musl) install(FILES ${CRTEND} ${CRTBEGIN} DESTINATION includeos/${ARCH}/lib) -install(FILES ${NEWLIB_LIB_DIR}/libc.a ${NEWLIB_LIB_DIR}/libg.a ${NEWLIB_LIB_DIR}/libm.a ${LIBGCC_LIB_DIR}/libgcc.a ${LIBCXX_LIB_DIR}/libc++.a ${LIBCXX_LIB_DIR}/libc++abi.a DESTINATION includeos/${ARCH}/lib) +install(FILES ${MUSL_LIB_DIR}/libc.a ${LIBCXX_LIB_DIR}/libc++.a ${LIBCXX_LIB_DIR}/libc++abi.a DESTINATION includeos/${ARCH}/lib) diff --git a/etc/build_llvm.sh b/etc/build_llvm.sh index 86843a669e..d63df2d61f 100755 --- a/etc/build_llvm.sh +++ b/etc/build_llvm.sh @@ -5,7 +5,7 @@ ARCH=${ARCH:-x86_64} # CPU architecture. Alternatively x86_64 TARGET=$ARCH-elf # Configure target based on arch. Always ELF. -newlib_inc=$TEMP_INSTALL_DIR/$TARGET/include # path for newlib headers +musl_inc=$TEMP_INSTALL_DIR/$TARGET/include # path for newlib headers IncludeOS_posix=$INCLUDEOS_SRC/api/posix libcxx_inc=$BUILD_DIR/llvm/projects/libcxx/include libcxxabi_inc=$BUILD_DIR/llvm/projects/libcxxabi/include @@ -19,7 +19,7 @@ download_llvm=${download_llvm:-"1"} # This should be more dynamic if [ ! -z $download_llvm ]; then # Clone LLVM - git clone -b $llvm_branch git@github.com:llvm-mirror/llvm.git || true + git clone -b $llvm_branch https://github.com/llvm-mirror/llvm.git || true #svn co http://llvm.org/svn/llvm-project/llvm/tags/$LLVM_TAG llvm # Clone libc++, libc++abi, and some extra stuff (recommended / required for clang) @@ -27,19 +27,19 @@ if [ ! -z $download_llvm ]; then git checkout $llvm_branch # Compiler-rt - git clone -b $llvm_branch git@github.com:llvm-mirror/compiler-rt.git || true + git clone -b $llvm_branch https://github.com/llvm-mirror/compiler-rt.git || true #svn co http://llvm.org/svn/llvm-project/compiler-rt/tags/$LLVM_TAG compiler-rt # libc++abi - git clone -b $llvm_branch git@github.com:llvm-mirror/libcxxabi.git || true + git clone -b $llvm_branch https://github.com/llvm-mirror/libcxxabi.git || true #svn co http://llvm.org/svn/llvm-project/libcxxabi/tags/$LLVM_TAG libcxxabi # libc++ - git clone -b $llvm_branch git@github.com:llvm-mirror/libcxx.git || true + git clone -b $llvm_branch https://github.com/llvm-mirror/libcxx.git || true #svn co http://llvm.org/svn/llvm-project/libcxx/tags/$LLVM_TAG libcxx # libunwind - git clone -b $llvm_branch git@github.com:llvm-mirror/libunwind.git || true + git clone -b $llvm_branch https://github.com/llvm-mirror/libunwind.git || true #svn co http://llvm.org/svn/llvm-project/libunwind/tags/$LLVM_TAG libunwind # Back to start @@ -64,7 +64,7 @@ fi TRIPLE=$ARCH-pc-linux-elf -CXX_FLAGS="-std=c++14 -msse3 -mfpmath=sse" +CXX_FLAGS="-std=c++14 -msse3 -mfpmath=sse -D_LIBCPP_HAS_MUSL_LIBC" # CMAKE configure step # @@ -75,31 +75,34 @@ CXX_FLAGS="-std=c++14 -msse3 -mfpmath=sse" echo "Building LLVM for $TRIPLE" - + # -DCMAKE_C_COMPILER=clang-$clang_version \ + # -DCMAKE_CXX_COMPILER=clang++-$clang_version \ cmake -GNinja $OPTS \ - -DCMAKE_CXX_FLAGS="$CXX_FLAGS -I$IncludeOS_posix -I$libcxxabi_inc -I$libcxx_inc -I$newlib_inc " $BUILD_DIR/llvm \ + -DCMAKE_CXX_FLAGS="$CXX_FLAGS -I$IncludeOS_posix -I$libcxxabi_inc -I$libcxx_inc -I$musl_inc " $BUILD_DIR/llvm \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_C_COMPILER=clang-$clang_version \ - -DCMAKE_CXX_COMPILER=clang++-$clang_version \ -DTARGET_TRIPLE=$TRIPLE \ -DLLVM_BUILD_32_BITS=OFF \ -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_ENABLE_THREADS=OFF \ + -DLLVM_ENABLE_THREADS=ON \ -DLLVM_DEFAULT_TARGET_TRIPLE=$TRIPLE \ -DLIBCXX_ENABLE_SHARED=OFF \ - -DLIBCXX_ENABLE_THREADS=OFF \ + -DLIBCXX_ENABLE_THREADS=ON \ -DLIBCXX_TARGET_TRIPLE=$TRIPLE \ -DLIBCXX_BUILD_32_BITS=OFF \ -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ -DLIBCXX_CXX_ABI=libcxxabi \ -DLIBCXXABI_TARGET_TRIPLE=$TRIPLE \ -DLIBCXXABI_ENABLE_THREADS=ON \ - -DLIBCXXABI_HAS_PTHREAD_API=ON + -DLIBCXXABI_HAS_PTHREAD_API=ON \ + -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ + -DLIBUNWIND_TARGET_TRIPLE=$TRIPLE \ + -DLIBUNWIND_ENABLE_SHARED=OFF # MAKE ninja libc++abi.a + ninja libc++.a popd diff --git a/etc/build_musl.sh b/etc/build_musl.sh new file mode 100755 index 0000000000..3732aa3945 --- /dev/null +++ b/etc/build_musl.sh @@ -0,0 +1,22 @@ +#! /bin/bash +. $INCLUDEOS_SRC/etc/set_traps.sh + +pushd $BUILD_DIR +MUSL_DIR="build_musl" + +# Download +if [ ! -d musl-$musl_version ]; then + echo -e "\n\n >>> Getting musl \n" + git clone git://git.musl-libc.org/musl/ || true +fi + + + +pushd musl +git checkout $musl_version +make distclean || true +# CC="$CC -m32" +export CFLAGS="$CFLAGS -target $ARCH-pc-linux-elf" +./configure --prefix=$TEMP_INSTALL_DIR/$TARGET --disable-shared --enable-optimize=* --target=$TARGET +make $num_jobs +make install diff --git a/etc/create_binary_bundle.sh b/etc/create_binary_bundle.sh index 4bc8412f48..4017a49f21 100755 --- a/etc/create_binary_bundle.sh +++ b/etc/create_binary_bundle.sh @@ -14,21 +14,19 @@ export TARGET=$ARCH-elf # Configure target based on arch. Always ELF. export num_jobs=${num_jobs:--j} # Specify number of build jobs # Version numbers -export binutils_version=${binutils_version:-2.28} # ftp://ftp.gnu.org/gnu/binutils -export newlib_version=${newlib_version:-2.5.0.20170323} # ftp://sourceware.org/pub/newlib -#export newlib_version=${newlib_version:-2.5.0} # ftp://sourceware.org/pub/newlib -export gcc_version=${gcc_version:-6.3.0} # ftp://ftp.nluug.nl/mirror/languages/gcc/releases/ +export binutils_version=${binutils_version:-2.28} #ftp://ftp.gnu.org/gnu/binutils +export musl_version=${musl_version:-v1.1.18} export clang_version=${clang_version:-3.9} # http://releases.llvm.org/ -export LLVM_TAG=${LLVM_TAG:-RELEASE_391/final} # http://llvm.org/svn/llvm-project/llvm/tags +#export LLVM_TAG=${LLVM_TAG:-RELEASE_391/final} # http://llvm.org/svn/llvm-project/llvm/tags export llvm_branch=${llvm_branch:-release_39} # Options to skip steps [ ! -v do_binutils ] && do_binutils=1 -[ ! -v do_gcc ] && do_gcc=1 -[ ! -v do_newlib ] && do_newlib=1 +[ ! -v do_musl ] && do_musl=1 [ ! -v do_includeos ] && do_includeos=1 [ ! -v do_llvm ] && do_llvm=1 [ ! -v do_bridge ] && do_bridge=1 +[ ! -v do_packages ] && do_packages=1 ############################################################ # COMMAND LINE PROPERTIES: @@ -67,11 +65,10 @@ printf " %-25s %-25s %s\n"\ "Env variable" "Description" "Value"\ "------------" "-----------" "-----"\ "INCLUDEOS_SRC" "Source dir of IncludeOS" "$INCLUDEOS_SRC"\ - "binutils_version" "binutils version" "$binutils_version"\ - "newlib_version" "newlib version" "$newlib_version"\ - "gcc_version" "gcc version" "$gcc_version"\ + "binutils_version" "binutils version" "$binutils_version"\ + "musl_version" "musl version" "$musl_version"\ "clang_version" "clang version" "$clang_version"\ - "LLVM_TAG" "LLVM version" "$LLVM_TAG"\ + "llvm_branch" "LLVM version" "$llvm_branch"\ "BUNDLE_ARCHES" "Build for CPU arches" "$BUNDLE_ARCHES"\ # Give user option to evaluate install options @@ -106,18 +103,13 @@ function do_build { echo -e "\n\n >>> Building bundle for ${ARCH} \n" # Build all sources if [ ! -z $do_binutils ]; then - echo -e "\n\n >>> GETTING / BUILDING binutils (Required for libgcc / unwind / crt) \n" + echo -e "\n\n >>> GETTING / BUILDING binutils (Required for cross compilation) \n" $INCLUDEOS_SRC/etc/build_binutils.sh fi - if [ ! -z $do_gcc ]; then - echo -e "\n\n >>> GETTING / BUILDING GCC COMPILER (Required for libgcc / unwind / crt) \n" - $INCLUDEOS_SRC/etc/build_gcc.sh - fi - - if [ ! -z $do_newlib ]; then - echo -e "\n\n >>> GETTING / BUILDING NEWLIB \n" - $INCLUDEOS_SRC/etc/build_newlib.sh + if [ ! -z $do_musl ]; then + echo -e "\n\n >>> GETTING / BUILDING MUSL \n" + $INCLUDEOS_SRC/etc/build_musl.sh fi if [ ! -z $do_llvm ]; then @@ -135,42 +127,32 @@ function do_build { echo ">>> Creating Installation Bundle as $BUNDLE_LOC" - newlib=$TEMP_INSTALL_DIR/$TARGET/lib + musl=$TEMP_INSTALL_DIR/$TARGET/lib llvm=$BUILD_DIR/build_llvm - # Libraries - libc=$newlib/libc.a - libm=$newlib/libm.a - libg=$newlib/libg.a + libcpp=$llvm/lib/libc++.a libcppabi=$llvm/lib/libc++abi.a - - GPP=$TEMP_INSTALL_DIR/bin/$TARGET-g++ - GCC_VER=`$GPP -dumpversion` - libgcc=$TEMP_INSTALL_DIR/lib/gcc/$TARGET/$GCC_VER/libgcc.a + libunwind=$llvm/lib/libunwind.a # Includes - include_newlib=$TEMP_INSTALL_DIR/$TARGET/include + include_musl=$TEMP_INSTALL_DIR/$TARGET/include include_libcxx=$llvm/include/c++/v1 # Make directory-tree mkdir -p $BUNDLE_LOC - mkdir -p $BUNDLE_LOC/newlib + mkdir -p $BUNDLE_LOC/musl mkdir -p $BUNDLE_LOC/libcxx - mkdir -p $BUNDLE_LOC/crt - mkdir -p $BUNDLE_LOC/libgcc + # Copy binaries cp $libcpp $BUNDLE_LOC/libcxx/ cp $libcppabi $BUNDLE_LOC/libcxx/ - cp $libm $BUNDLE_LOC/newlib/ - cp $libc $BUNDLE_LOC/newlib/ - cp $libg $BUNDLE_LOC/newlib/ - cp $libgcc $BUNDLE_LOC/libgcc/ - cp $TEMP_INSTALL_DIR/lib/gcc/$TARGET/$GCC_VER/crt*.o $BUNDLE_LOC/crt/ + cp $libunwind $BUNDLE_LOC/libcxx/ + cp -r $musl $BUNDLE_LOC/musl/ # Copy includes - cp -r $include_newlib $BUNDLE_LOC/newlib/ + cp -r $include_musl $BUNDLE_LOC/musl/ cp -r $include_libcxx $BUNDLE_LOC/libcxx/include } diff --git a/lib/LiveUpdate/CMakeLists.txt b/lib/LiveUpdate/CMakeLists.txt index 62b426eb59..31243b8ed9 100644 --- a/lib/LiveUpdate/CMakeLists.txt +++ b/lib/LiveUpdate/CMakeLists.txt @@ -5,7 +5,7 @@ add_definitions(-DARCH="${ARCH}") include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) -include_directories(${NEWLIB_INCLUDE_DIR}) +include_directories(${MUSL_INCLUDE_DIR}) include_directories(${INCLUDEOS_ROOT}/src/include) include_directories(${INCLUDEOS_ROOT}/api) include_directories(${INCLUDEOS_ROOT}/mod/GSL/) diff --git a/lib/mana/CMakeLists.txt b/lib/mana/CMakeLists.txt index 9619fe26f2..be04deaa9d 100644 --- a/lib/mana/CMakeLists.txt +++ b/lib/mana/CMakeLists.txt @@ -9,7 +9,7 @@ include_directories(${INCLUDEOS_ROOT}/mod/rapidjson/include) include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) -include_directories(${NEWLIB_INCLUDE_DIR}) +include_directories(${MUSL_INCLUDE_DIR}) include_directories(${INCLUDEOS_ROOT}/src/include) include_directories(${INCLUDEOS_ROOT}/api) include_directories(${INCLUDEOS_ROOT}/mod/GSL/) diff --git a/lib/mender/CMakeLists.txt b/lib/mender/CMakeLists.txt index 31052216eb..8b715be075 100644 --- a/lib/mender/CMakeLists.txt +++ b/lib/mender/CMakeLists.txt @@ -5,7 +5,7 @@ add_definitions(-DARCH="${ARCH}") include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) -include_directories(${NEWLIB_INCLUDE_DIR}) +include_directories(${MUSL_INCLUDE_DIR}) include_directories(${INCLUDEOS_ROOT}/src/include) include_directories(${INCLUDEOS_ROOT}/api) include_directories(${INCLUDEOS_ROOT}/mod/GSL/) diff --git a/lib/microLB/CMakeLists.txt b/lib/microLB/CMakeLists.txt index d86a7daccf..53983f1827 100644 --- a/lib/microLB/CMakeLists.txt +++ b/lib/microLB/CMakeLists.txt @@ -12,7 +12,7 @@ add_library(microlb STATIC target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/api/posix) target_include_directories(microlb PUBLIC ${LIBCXX_INCLUDE_DIR}) -target_include_directories(microlb PUBLIC ${NEWLIB_INCLUDE_DIR}) +target_include_directories(microlb PUBLIC ${MUSL_INCLUDE_DIR}) target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/src/include) target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/api) target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/lib/LiveUpdate) diff --git a/lib/uplink/CMakeLists.txt b/lib/uplink/CMakeLists.txt index 71864d64b9..6bf6b31a74 100644 --- a/lib/uplink/CMakeLists.txt +++ b/lib/uplink/CMakeLists.txt @@ -5,7 +5,7 @@ add_definitions(-DARCH="${ARCH}") include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) -include_directories(${NEWLIB_INCLUDE_DIR}) +include_directories(${MUSL_INCLUDE_DIR}) include_directories(${INCLUDEOS_ROOT}/src/include) include_directories(${INCLUDEOS_ROOT}/api) include_directories(${INCLUDEOS_ROOT}/mod/GSL/) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ea3dfd20db..ff84415298 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,7 +9,7 @@ endif() include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) -include_directories(${NEWLIB_INCLUDE_DIR}) +include_directories(${MUSL_INCLUDE_DIR}) include_directories(${SOLO5_INCLUDE_DIR}) include_directories(${INCLUDEOS_ROOT}/src/include) include_directories(${INCLUDEOS_ROOT}/api) @@ -58,12 +58,7 @@ set(OS_OBJECTS net/nat/nat.cpp net/nat/napt.cpp fs/disk.cpp fs/filesystem.cpp fs/dirent.cpp fs/mbr.cpp fs/path.cpp fs/fat.cpp fs/fat_async.cpp fs/fat_sync.cpp fs/memdisk.cpp - posix/fd.cpp posix/tcp_fd.cpp posix/udp_fd.cpp posix/unistd.cpp posix/fcntl.cpp posix/syslog.cpp - posix/sys/socket.cpp posix/sys/select.cpp posix/sys/utsname.cpp posix/sys/mman.cpp posix/arpa/inet.cpp - posix/ucontext.cpp posix/ucontext_asm.asm posix/file_fd.cpp posix/dlfcn.cpp - posix/sys/stat.cpp posix/sys/ioctl.cpp posix/ftw.cpp posix/utime.cpp posix/math.cpp - posix/pwd.cpp posix/pthread.cpp posix/poll.cpp - posix/signal.cpp posix/stdio.cpp posix/dirent.cpp posix/termios.cpp) +) add_library(os STATIC ${OS_OBJECTS}) add_dependencies(os PrecompiledLibraries botan ${OPENSSL_LIBS}) diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index 95742cceed..1a9663be40 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -24,7 +24,7 @@ #include #include #include -#include +#include #include #if __LP64__ diff --git a/src/posix/arpa/inet.cpp b/src/posix/arpa/inet.cpp deleted file mode 100644 index c3ceeba302..0000000000 --- a/src/posix/arpa/inet.cpp +++ /dev/null @@ -1,115 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -/** - * @note: shortcut, not sufficent. - * see: http://pubs.opengroup.org/onlinepubs/9699919799/functions/inet_addr.html# - */ -in_addr_t inet_addr(const char* cp) -{ - try { - const auto addr = net::ip4::Addr{cp}; - return addr.whole; - } - catch(const net::ip4::Invalid_address&) { - return (in_addr_t)(-1); - } -} - -char* inet_ntoa(struct in_addr ina) -{ - static char buffer[INET_ADDRSTRLEN]; - unsigned char* byte = (unsigned char *)&ina; - - sprintf(buffer, "%hhu.%hhu.%hhu.%hhu", - byte[0], byte[1], byte[2], byte[3]); - return buffer; -} - -/** - * @note: Missing IPv6 support - */ -const char* inet_ntop(int af, const void *__restrict__ src, char *__restrict__ dst, socklen_t size) -{ - if(UNLIKELY(dst == nullptr)) - return nullptr; - - // IPv4 - if(af == AF_INET) - { - if(LIKELY(size >= INET_ADDRSTRLEN)) - { - unsigned char* byte = (unsigned char*)src; - - sprintf(dst, "%hhu.%hhu.%hhu.%hhu", - byte[0], byte[1], byte[2], byte[3]); - - return dst; - } - else - { - errno = ENOSPC; - } - } - else if(af == AF_INET6) - { - // add me ;( - errno = EAFNOSUPPORT; - } - else - { - errno = EAFNOSUPPORT; - } - return nullptr; -} - -/** - * @note: Missing IPv6 support - */ -int inet_pton(int af, const char *__restrict__ src, void *__restrict__ dst) -{ - // IPv4 - if(af == AF_INET) - { - try - { - const auto addr = net::ip4::Addr{src}; - memcpy(dst, &addr.whole, INET_ADDRSTRLEN); - return 1; - } - catch(const net::ip4::Invalid_address&) - { - return 0; - } - } - else if(af == AF_INET6) - { - // add me ;( - errno = EAFNOSUPPORT; - } - else - { - errno = EAFNOSUPPORT; - } - return -1; - -} diff --git a/src/posix/dirent.cpp b/src/posix/dirent.cpp deleted file mode 100644 index 80c6c2c0b6..0000000000 --- a/src/posix/dirent.cpp +++ /dev/null @@ -1,40 +0,0 @@ -#include -#include -#include -#include - -struct DIR -{ - ino_t d_ino; // file serial number - char d_name[]; // name of entry -}; - -struct dirent { - ino_t d_ino; /* inode number */ - off_t d_off; /* offset to the next dirent */ - unsigned short d_reclen; /* length of this record */ - unsigned char d_type; /* type of file; not supported - by all file system types */ - char d_name[256]; /* filename */ -}; - -extern "C" -DIR* opendir(const char *dirname) -{ - PRINT("opendir(%s) = -1\n", dirname); - errno = ENOENT; - return nullptr; -} -extern "C" -struct dirent* readdir(DIR *dirp) -{ - PRINT("readdir(%p) = -1\n", dirp); - errno = EINVAL; - return nullptr; -} -extern "C" -int closedir(DIR *dirp) -{ - PRINT("closedir(%p) = 0\n", dirp); - return 0; -} diff --git a/src/posix/dlfcn.cpp b/src/posix/dlfcn.cpp deleted file mode 100644 index 316bdf8fcf..0000000000 --- a/src/posix/dlfcn.cpp +++ /dev/null @@ -1,57 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -void* dlopen(const char *filename, int flag) -{ - printf("dlopen called for %s with flag %#x\n", filename, flag); - return nullptr; -} -char* dlerror(void) -{ - printf("dlerror\n"); - return nullptr; -} -void* dlsym(void*, const char* symbol) -{ - printf("dlsym on %s\n", symbol); - return nullptr; -} -int dlclose(void*) -{ - printf("dlclose\n"); - return 0; -} - -typedef struct { - const char *dli_fname; /* Pathname of shared object that - contains address */ - void *dli_fbase; /* Base address at which shared - object is loaded */ - const char *dli_sname; /* Name of symbol whose definition - overlaps addr */ - void *dli_saddr; /* Exact address of symbol named - in dli_sname */ -} Dl_info; - -extern "C" -int dladdr(void*, Dl_info*) -{ - return 0; -} diff --git a/src/posix/fcntl.cpp b/src/posix/fcntl.cpp deleted file mode 100644 index 389986a928..0000000000 --- a/src/posix/fcntl.cpp +++ /dev/null @@ -1,56 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include - -int creat(const char* filename, mode_t mode) -{ - PRINT("creat(%s, %x) = -1\n", filename, mode); - errno = EINVAL; - return -1; -} -int fcntl(int fd, int cmd, ... /* arg */ ) -{ - PRINT("fcntl(%d, %d, ...)\n", fd, cmd); - try { - auto& desc = FD_map::_get(fd); - va_list va; - va_start(va, cmd); - int ret = desc.fcntl(cmd, va); - va_end(va); - return ret; - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -int posix_fadvise(int fd, off_t, off_t, int) -{ - PRINT("posix_fadvise(%d) = -1\n", fd); - return -1; -} -int posix_fallocate(int fd, off_t, off_t) -{ - PRINT("opendir(%d) = -1\n", fd); - return -1; -} diff --git a/src/posix/fd.cpp b/src/posix/fd.cpp deleted file mode 100644 index 138afda1b5..0000000000 --- a/src/posix/fd.cpp +++ /dev/null @@ -1,60 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include - -int FD::fcntl(int cmd, va_list list) -{ - PRINT("fcntl(%d)\n", cmd); - switch (cmd) { - case F_GETFD: - // return descriptor flags - return dflags; - case F_SETFD: - // set desc flags from va_list - dflags = va_arg(list, int); - return 0; - case F_GETFL: - // return file access flags - return fflags; - case F_SETFL: - // set file access flags - fflags = va_arg(list, int); - return 0; - case F_DUPFD: - case F_DUPFD_CLOEXEC: - default: - errno = EINVAL; - return -1; - } -} -int FD::getsockopt(int fd, int, void *__restrict__, socklen_t *__restrict__) -{ - PRINT("getsockopt(%d) = -1\n", fd); - errno = ENOTSOCK; - return -1; -} -int FD::setsockopt(int fd, int, const void *, socklen_t) -{ - PRINT("setsockopt(%d) = -1\n", fd); - errno = ENOTSOCK; - return -1; -} diff --git a/src/posix/file_fd.cpp b/src/posix/file_fd.cpp deleted file mode 100644 index c9d7403f3a..0000000000 --- a/src/posix/file_fd.cpp +++ /dev/null @@ -1,59 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -int File_FD::read(void* p, size_t n) { - auto buf = ent_.read(offset_, n); - memcpy(p, buf.data(), n); - offset_ += buf.size(); - return buf.size(); -} - -int File_FD::write(const void*, size_t) { - return -1; -} - -int File_FD::close() { - return 0; -} - -int File_FD::lseek(off_t offset, int whence) -{ - if ((whence != SEEK_SET) && (whence != SEEK_CUR) && (whence != SEEK_END)) { - PRINT("lseek(%lu, %d) == %d\n", offset, whence, -1); - errno = EINVAL; - return -1; - } - off_t calculated_offset = offset_; - if (whence == SEEK_SET) { - calculated_offset = offset; - } - else if (whence == SEEK_CUR) { - calculated_offset += offset; - } - else if (whence == SEEK_END) { - const off_t end = ent_.size(); - calculated_offset = end + offset; - } - if (calculated_offset < 0) calculated_offset = 0; - offset_ = calculated_offset; - PRINT("lseek(%lu, %d) == %d\n", offset, whence, offset_); - return offset_; -} diff --git a/src/posix/ftw.cpp b/src/posix/ftw.cpp deleted file mode 100644 index 6d178ac2f9..0000000000 --- a/src/posix/ftw.cpp +++ /dev/null @@ -1,99 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include - -extern const std::string& cwd_ref(); - -class Walker { -public: - explicit Walker(Nftw_func fn, int flags) : fn_ptr_ {fn}, flags_ {flags} {}; - - int walk(const std::string& path, int level = 0) noexcept { - struct stat buffer {}; - struct FTW ftw {}; - ftw.level = level; - ftw.base = path.find_last_of('/') + 1; - - std::string abs_path; - if (path.front() != '/') { - abs_path = cwd_ref() + "/" + path; - } - else - { - abs_path = path; - } - - int res = stat(abs_path.c_str(), &buffer); - if (res != -1) { - if (S_ISDIR(buffer.st_mode) == 0) { - // path is a file - return fn_ptr_(abs_path.c_str(), &buffer, FTW_F, &ftw); - } - else { - // path is a directory - int result; - if (not (flags_ & FTW_DEPTH)) { - // call fn on dir first - result = fn_ptr_(abs_path.c_str(), &buffer, FTW_D, &ftw); - } - // call fn on each entry - ++level; - auto ent = fs::VFS::stat_sync(abs_path); - ent.ls(fs::on_ls_func::make_packed( - [abs_path, &result, &level, this](auto, auto entries) - { - for (auto&& ent : *entries) - { - if (ent.name() == "." or ent.name() == "..") - { - //printf("Skipping %s\n", ent.name().c_str()); - } - else if ((result = this->walk(abs_path + "/" + ent.name(), level) != 0)) - break; - } - }) - ); - if (flags_ & FTW_DEPTH) { - result = fn_ptr_(abs_path.c_str(), &buffer, FTW_D, &ftw); - } - return result; - } - } - else { - // could not stat path - return fn_ptr_(abs_path.c_str(), &buffer, FTW_NS, &ftw); - } - } -private: - Nftw_func* fn_ptr_; - int flags_; -}; - -int nftw(const char* path, Nftw_func fn, int fd_limit, int flags) { - (void) fd_limit; - if (path == nullptr || fn == nullptr) - { - return EFAULT; - } - Walker walker {fn, flags}; - return walker.walk(path); -} diff --git a/src/posix/math.cpp b/src/posix/math.cpp deleted file mode 100644 index f46abded54..0000000000 --- a/src/posix/math.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include -#include - -double trunc(double v) { - return (double) (int64_t) v; -} diff --git a/src/posix/poll.cpp b/src/posix/poll.cpp deleted file mode 100644 index 62a9f09e2a..0000000000 --- a/src/posix/poll.cpp +++ /dev/null @@ -1,46 +0,0 @@ -#include -#include -#include -#include - -#define POLLIN 0x0001 -#define POLLPRI 0x0002 -#define POLLOUT 0x0004 -#define POLLERR 0x0008 -#define POLLHUP 0x0010 -#define POLLNVAL 0x0020 - -struct pollfd { - int fd; /* file descriptor */ - short events; /* requested events */ - short revents; /* returned events */ -}; -typedef unsigned long nfds_t; - -extern "C" -int poll(struct pollfd *fds, nfds_t nfds, int timeout) -{ - PRINT("poll(%p, %lu, %d)\n", fds, nfds, timeout); - if (nfds == 1 && fds[0].fd == 4) - { - //PRINT("poll on random device\n"); - fds[0].revents = fds[0].events; - return 1; - } - - if (fds == nullptr) return -1; - if (timeout < 0) return -1; - - bool still_waiting = true; - do { - for (unsigned i = 0; i < nfds; i++) - { - auto& pfd = fds[i]; - printf("polling %d for events %d\n", pfd.fd, pfd.events); - /* code */ - - } - break; - } while(still_waiting); - return 0; -} diff --git a/src/posix/pthread.cpp b/src/posix/pthread.cpp deleted file mode 100644 index 0a64ae2bcf..0000000000 --- a/src/posix/pthread.cpp +++ /dev/null @@ -1,188 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include - -std::unordered_map fibers; - -int pthread_create(pthread_t* th, const pthread_attr_t* attr, void *(*func)(void *), void* args) { - - static int __thread_id__ = 0; - auto new_id = __thread_id__++; - - fibers.emplace(new_id, Fiber{func, args}); - fibers[new_id].start(); - - *th = new_id; - return 0; -} - -int pthread_join(pthread_t thread, void **value_ptr) { - *value_ptr = fibers[thread].ret(); - return 0; -} - - -int sched_yield() -{ - SMP_PRINT("WARNING: Sched yield called\n"); -} - -thread_local pthread_t tself = 0; -pthread_t pthread_self() -{ - return tself; -} - -int pthread_detach(pthread_t th) -{ - printf("pthread_detach %d\n", th); - return 0; -} -int pthread_equal(pthread_t t1, pthread_t t2) -{ - return t1 == t2; -} - - -int pthread_mutex_init(pthread_mutex_t* mutex, const pthread_mutexattr_t* attr) -{ - if (mutex == nullptr) return 1; - mutex->spinlock = 0; - return 0; -} -int pthread_mutex_lock(pthread_mutex_t *mutex) -{ - if (mutex == nullptr) return 1; - lock(mutex->spinlock); - //SMP_PRINT("Spin locked %p\n", mutex); - return 0; -} -int pthread_mutex_trylock(pthread_mutex_t *mutex) -{ - if (mutex == nullptr) return 1; - if (mutex->spinlock) return 1; - lock(mutex->spinlock); - return 0; -} -int pthread_mutex_unlock(pthread_mutex_t *mutex) -{ - if (mutex == nullptr) return 1; - //SMP_PRINT("Spin unlocked %p\n", mutex); - unlock(mutex->spinlock); - return 0; -} -int pthread_mutex_destroy(pthread_mutex_t *mutex) -{ - return 0; -} - -int pthread_once(pthread_once_t*, void (*routine)()) -{ - thread_local bool executed = false; - if (!executed) { - //SMP_PRINT("Executing pthread_once routine %p\n", routine); - executed = true; - routine(); - } - return 0; -} - -int pthread_cond_wait(pthread_cond_t* cond, pthread_mutex_t* mutex) -{ - //SMP_PRINT("pthread_cond_wait\n"); - return 0; -} -int pthread_cond_broadcast(pthread_cond_t* cond) -{ - //SMP_PRINT("pthread_cond_broadcast\n"); - return 0; -} -int pthread_cond_signal(pthread_cond_t* cond) -{ - //SMP_PRINT("pthread_cond_signal %p\n", cond); - return 0; -} -int pthread_cond_timedwait(pthread_cond_t* cond, - pthread_mutex_t* mutex, - const struct timespec* abstime) -{ - SMP_PRINT("pthread_cond_timedwait cond %p mutex %p\n", cond, mutex); - return 0; -} -int pthread_cond_destroy(pthread_cond_t *cond) -{ - SMP_PRINT("pthread_cond_timedwait cond %p\n", cond); - return 0; -} - - -std::vector key_vec; -spinlock_t key_lock = 0; - -void* pthread_getspecific(pthread_key_t key) -{ - scoped_spinlock spinlock(key_lock); - return (void*) key_vec.at(key); -} -int pthread_setspecific(pthread_key_t key, const void *value) -{ - scoped_spinlock spinlock(key_lock); - key_vec.at(key) = value; - return 0; -} -int pthread_key_create(pthread_key_t *key, void (*destructor)(void*)) -{ - //SMP_PRINT("pthread_key_create: %p destructor %p\n", key, destructor); - scoped_spinlock spinlock(key_lock); - key_vec.push_back(nullptr); - *key = key_vec.size()-1; - return 0; -} - -int pthread_mutexattr_destroy(pthread_mutexattr_t *attr) -{ - PRINT("pthread_mutexattr_destroy(%p) = 0\n", attr); - return 0; -} -int pthread_mutexattr_init(pthread_mutexattr_t *attr) -{ - PRINT("pthread_mutexattr_init(%p) = 0\n", attr); - return 0; -} -int pthread_mutexattr_gettype(const pthread_mutexattr_t *__restrict attr, int *__restrict type) -{ - return 0; -} -int pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) -{ - return 0; -} - -extern "C" -int nanosleep(const struct timespec *req, struct timespec *rem) -{ - PRINT("nanosleep(%p, %p) = -1\n", req, rem); - return -1; -} diff --git a/src/posix/pwd.cpp b/src/posix/pwd.cpp deleted file mode 100644 index 0dba19fa2e..0000000000 --- a/src/posix/pwd.cpp +++ /dev/null @@ -1,168 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include - -const char* pwd_path = "/etc/passwd"; - -/* Applications are not allowed to modify the structure to which the return - values point, nor any storage areas pointed to by pointers within the - structure. Returned pointers might be invalidated or the structure/ - storage areas might be overwritten by subsequent calls. */ -static struct passwd ret; - -class Pwd { -public: - static Pwd& instance() { - static Pwd pwd; - return pwd; - } - - void rewind() {pos_ = 0;} - - void close() {open_ = false;} - - struct passwd* find(const char* name) { - Expects(name != nullptr); - if (!open_) { - read(); - } - const auto it = std::find_if(begin(entries_), end(entries_), [name](const auto& entry){ - return (strcmp(name, entry.c_str()) == 0); - }); - if (it == end(entries_)) { - return nullptr; - } - extract(*it); - return &ret; - } - - struct passwd* find(uid_t uid) { - char buf[16]; - snprintf(buf, 16, "%d", uid); - if (!open_) { - read(); - } - const auto it = std::find_if(begin(entries_), end(entries_), [buf](const auto& entry){ - char* start = const_cast(entry.c_str()); - size_t pw_pos = entry.find('\0') + 1; - size_t uid_pos = entry.find('\0', pw_pos) + 1; - return (strcmp(buf, start + uid_pos) == 0); - }); - if (it == end(entries_)) { - return nullptr; - } - extract(*it); - return &ret; - } - - struct passwd* next() noexcept { - if (!open_) { - read(); - } - if (pos_ >= entries_.size()) { - return nullptr; - } - try { - const auto& ent = entries_[pos_]; - int field_seps = std::count(std::begin(ent), std::end(ent), '\0'); - if (field_seps != 6) { - INFO("pwd", "not a valid passwd file entry"); - return nullptr; - } - extract(ent); - ++pos_; - return &ret; - } - catch (...) { - INFO("pwd", "error parsing pwd entry"); - return nullptr; - } - } - - void extract(const std::string& ent) { - ret.pw_name = const_cast(ent.c_str()); - size_t pw_pos = ent.find('\0') + 1; - size_t uid_pos = ent.find('\0', pw_pos) + 1; - size_t gid_pos = ent.find('\0', uid_pos) + 1; - size_t info_pos = ent.find('\0', gid_pos) + 1; - size_t dir_pos = ent.find('\0', info_pos) + 1; - size_t shell_pos = ent.find('\0', dir_pos) + 1; - ret.pw_uid = std::stol(ret.pw_name + uid_pos); - ret.pw_gid = std::stol(ret.pw_name + gid_pos); - ret.pw_dir = ret.pw_name + dir_pos; - ret.pw_shell = ret.pw_name + shell_pos; - } - - void print() { - printf(" instance @%p\n", this); - printf(" open: %d\n", open_); - printf(" pos: %d\n", pos_); - printf(" entries: %d\n", entries_.size()); - } - -private: - Pwd() : open_ {false}, pos_ {0} {} - - void read() { - entries_.clear(); - std::ifstream is(pwd_path); - std::string line; - while (std::getline(is, line)) { - std::replace(std::begin(line), std::end(line), ':', '\0'); - entries_.push_back(line); - } - rewind(); - open_ = true; - } - bool open_; - size_t pos_; - std::vector entries_; -}; - -void endpwent(void) { - Pwd& pwd = Pwd::instance(); - pwd.close(); -} - -void setpwent(void) { - Pwd& pwd = Pwd::instance(); - pwd.rewind(); -} - -struct passwd *getpwent(void) { - Pwd& pwd = Pwd::instance(); - return pwd.next(); -} - -struct passwd *getpwnam(const char *name) { - if (name == nullptr) { - return nullptr; - } - Pwd& pwd = Pwd::instance(); - return pwd.find(name); -} - -struct passwd *getpwuid(uid_t uid) { - Pwd& pwd = Pwd::instance(); - return pwd.find(uid); -} diff --git a/src/posix/signal.cpp b/src/posix/signal.cpp deleted file mode 100644 index e726f9e771..0000000000 --- a/src/posix/signal.cpp +++ /dev/null @@ -1,16 +0,0 @@ -#include -#include -#include - -int sigaction(int signum, - const struct sigaction* act, - struct sigaction* oldact) -{ - if (signum == SIGKILL || signum == SIGSTOP) { - errno = EINVAL; - return -1; - } - printf("sigaction(%d, %p, %p)\n", signum, act, oldact); - *oldact = *act; - return 0; -} diff --git a/src/posix/stdio.cpp b/src/posix/stdio.cpp deleted file mode 100644 index 9a13f6920e..0000000000 --- a/src/posix/stdio.cpp +++ /dev/null @@ -1,71 +0,0 @@ -#include -#include -#include - -extern "C" -FILE* fopen64(const char *filename, const char *type) -{ - return fopen(filename, type); -} - -extern "C" __attribute__((format(printf, 2, 3))) -int __printf_chk (int flag, const char *format, ...) -{ - va_list ap; - va_start (ap, format); - int done = vfprintf (stdout, format, ap); - va_end (ap); - return done; -} -extern "C" -int __fprintf_chk(FILE* fp, int flag, const char* format, ...) -{ - va_list arg; - va_start (arg, format); - int done = vfprintf(fp, format, arg); - va_end (arg); - return done; -} -extern "C" -int __vfprintf_chk(FILE* fp, int flag, const char *format, va_list ap) -{ - int done; - done = vfprintf (fp, format, ap); - return done; -} -extern "C" __attribute__((format(printf, 4, 5))) -int __sprintf_chk(char* s, int flags, size_t slen, const char *format, ...) -{ - va_list arg; - int done; - va_start (arg, format); - done = vsprintf(s, format, arg); - va_end (arg); - return done; -} -extern "C" -char* __strcat_chk(char* dest, const char* src, size_t destlen) -{ - size_t len = strlen(dest) + strlen(src); - assert (len > destlen); - return strcat(dest, src); -} -extern "C" -int __isoc99_scanf (const char *format, ...) -{ - va_list arg; - va_start (arg, format); - int done = vfscanf(stdin, format, arg); - va_end (arg); - return done; -} -extern "C" __attribute__((format(scanf, 2, 3))) -int __isoc99_sscanf (const char *s, const char *format, ...) -{ - va_list arg; - int done; - va_start (arg, format); - done = vsscanf(s, format, arg); - va_end (arg); - return done; -} diff --git a/src/posix/sys/ioctl.cpp b/src/posix/sys/ioctl.cpp deleted file mode 100644 index e043cef35c..0000000000 --- a/src/posix/sys/ioctl.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include -#include - -int ioctl(int fd, unsigned long request, ...) -{ - printf("ioctl(%d, %x)\n", fd, request); -} diff --git a/src/posix/sys/mman.cpp b/src/posix/sys/mman.cpp deleted file mode 100644 index e239b3e8ee..0000000000 --- a/src/posix/sys/mman.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include - -#include -#include -#include -#include -#include -#include -#include - -struct mmap_entry_t -{ - void* addr; - size_t length; - int prot; - int flags; - int fd; - off_t offset; -}; -std::map _mmap_entries; - -void* mmap(void* addr, size_t length, - int prot, int flags, - int fd, off_t offset) -{ - PRINT("mmap(%p, %lu, %d, %d, %d, %lu)\n", - addr, length, prot, flags, fd, offset); - - // invalid or misaligned length - if (length == 0 || (length & 4095) != 0) - { - errno = EINVAL; - return MAP_FAILED; - } - - // TODO: - // validate fd is open file - - // associate some VA space with open file @fd - // for now just allocate page-aligned on heap - mmap_entry_t entry; - entry.addr = aligned_alloc(4096, length); - entry.length = length; - entry.prot = prot; - entry.flags = flags; - entry.fd = fd; - entry.offset = offset; - printf("mmap allocated %d bytes (%u pages)\n", - length, (uint32_t) (length / 4096)); - - // TODO: - // read entire file into entry space - // deallocate entry space on failure - - // create the mapping - _mmap_entries[addr] = entry; - return entry.addr; -} - -int munmap(void* addr, size_t length) -{ - PRINT("munmap(%p, %lu)\n", addr, length); - auto it = _mmap_entries.find(addr); - if (it != _mmap_entries.end()) - { - // - assert(it->second.length == length); - - // free and remove the entry - free(it->second.addr); - _mmap_entries.erase(it); - return 0; - } - errno = EINVAL; - return -1; -} - -void *mremap(void *old_address, size_t old_size, - size_t new_size, int flags, ... /* void *new_address */) -{ - PRINT("mremap(%p, %lu, %lu, %#x)\n", old_address, old_size, new_size, flags); - return 0; -} - -int mprotect(void *addr, size_t len, int prot) -{ - PRINT("mprotect(%p, %lu, %#x)\n", addr, len, prot); - return 0; -} diff --git a/src/posix/sys/select.cpp b/src/posix/sys/select.cpp deleted file mode 100644 index 275e052714..0000000000 --- a/src/posix/sys/select.cpp +++ /dev/null @@ -1,168 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include // OS::block() -#include - -static struct { - typedef std::pair listpair; - std::list list; - std::list conn; - - void clean() { - // reset callbacks on all listeners - for (auto& l : list) { - l.second.get_listener().on_connect([] (auto) {}); - } - list.clear(); - // reset callbacks on all connections - for (auto c : conn) { - c->reset_callbacks(); - } - conn.clear(); - } - -} temp; - -static int process_fds( - int max_fd, - fd_mask* bits, - delegate callback) -{ - for (size_t i = 0; i < sizeof (fd_set) / sizeof(fd_mask); i++) - { - int fd = sizeof(fd_mask) * i * 8; - fd_mask mask = bits[i]; - while (mask) - { - int lbits = __builtin_ctzl(mask); - fd += lbits; - // process descriptor - int ret = callback(fd); - if (ret) return ret; - - mask >>= lbits + 1; - if (++fd >= max_fd) return 0; - } - if (fd >= max_fd) return 0; - } - return 0; -} - -int select(int max_fd, - fd_set *__restrict__ reads, - fd_set *__restrict__ writes, - fd_set *__restrict__ excepts, - struct timeval *__restrict__ tv) -{ - PRINT("select(%d, %p, %p, %p)\n", max_fd, reads, writes, excepts); - // monitor file descriptors given in read, write and exceptional fd sets - int mon_read = -1; - int mon_send = -1; - int mon_err = -1; - int timer = -1; - bool timeout = false; - int ret = 0; - - if (reads) - { - ret = process_fds(max_fd, reads->fds_bits, - [] (int fd) -> int { - - try { - auto& desc = FD_map::_get(fd); - if (desc.is_socket()) { - auto& tcp = (TCP_FD&) desc; - if (tcp.is_listener()) { - /// monitor for new connections - temp.list.push_back({fd, tcp}); - } - else if (tcp.is_connection()) { - /// monitor for reads - } - } - return 0; - - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } - - }); - if (ret) return ret; - // clear read fds - FD_ZERO(reads); - } - if (writes) - { - ret = process_fds(max_fd, writes->fds_bits, - [&mon_send] (int) -> int { - errno = ENOTSUP; - return -1; - }); - if (ret) return ret; - // clear send fds - FD_ZERO(writes); - } - if (excepts) - { - ret = process_fds(max_fd, excepts->fds_bits, - [&mon_err] (int) -> int { - errno = ENOTSUP; - return -1; - }); - if (ret) return ret; - // clear exception fds - FD_ZERO(excepts); - } - // timeout - int64_t micros = tv->tv_sec * 1000000 + tv->tv_usec; - if (micros > 0) { - timer = Timers::oneshot( - std::chrono::microseconds(micros), - [&timeout] (auto) { - timeout = true; - }); - } - /// wait for something to happen - do { - for (auto& list : temp.list) { - if (list.second.has_connq()) { - mon_read = list.first; // fd - break; - } - } - - OS::block(); - } while (mon_read == -1 - && mon_send == -1 - && mon_err == -1 - && timeout == false); - - if (mon_read >= 0) FD_SET(mon_read, reads); - if (mon_send >= 0) FD_SET(mon_send, writes); - if (mon_err >= 0) FD_SET(mon_err, excepts); - if (timer >= 0) { - if (timeout == false) Timers::stop(timer); - } - // clear all the temp stuff - temp.clean(); - // all good - return 0; -} -int pselect(int max_fd, - fd_set *__restrict__ reads, - fd_set *__restrict__ writes, - fd_set *__restrict__ excepts, - const struct timespec *__restrict__ ts, - const sigset_t *__restrict__) -{ - PRINT("pselect(%d, %p, %p, %p)\n", max_fd, reads, writes, excepts); - struct timeval tv; - tv.tv_sec = ts->tv_sec; - tv.tv_usec = ts->tv_nsec / 1000; - return select(max_fd, reads, writes, excepts, &tv); -} diff --git a/src/posix/sys/socket.cpp b/src/posix/sys/socket.cpp deleted file mode 100644 index 1cc28aa1e5..0000000000 --- a/src/posix/sys/socket.cpp +++ /dev/null @@ -1,186 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include - -int socket(int domain, int type, int protocol) -{ - PRINT("socket(%d, %d, %d)\n", domain, type, protocol); - // disallow strange domains, like ALG - if (domain < 0 || domain > AF_INET6) { errno = EAFNOSUPPORT; return -1; } - // disallow RAW etc - if (type < 0 || type > SOCK_DGRAM) { errno = EINVAL; return -1; } - // we are purposefully ignoring the protocol argument - if (protocol < 0) { errno = EPROTONOSUPPORT; return -1; } - - return [](const int type)->int{ - switch(type) - { - case SOCK_STREAM: - return FD_map::_open().get_id(); - case SOCK_DGRAM: - return FD_map::_open().get_id(); - default: - errno = EINVAL; - return -1; - } - }(type); -} - -int connect(int socket, const struct sockaddr *address, socklen_t len) -{ - PRINT("connect(%d, %p, %d)\n", socket, address, len); - try { - auto& fd = FD_map::_get(socket); - return fd.connect(address, len); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -ssize_t send(int socket, const void *message, size_t len, int fmt) -{ - PRINT("send(%d, %p, %lu, %d)\n", socket, message, len, fmt); - try { - auto& fd = FD_map::_get(socket); - return fd.send(message, len, fmt); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} -ssize_t sendto(int socket, const void *message, size_t len, int flags, - const struct sockaddr *dest_addr, socklen_t dest_len) -{ - PRINT("sendto(%d, %p, %lu, %d, %p, %d)\n", socket, message, len, flags, dest_addr, dest_len); - try { - auto& fd = FD_map::_get(socket); - return fd.sendto(message, len, flags, dest_addr, dest_len); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} -ssize_t recv(int socket, void *buffer, size_t length, int flags) -{ - PRINT("recv(%d, %p, %lu, %d)\n", socket, buffer, length, flags); - try { - auto& fd = FD_map::_get(socket); - return fd.recv(buffer, length, flags); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} -ssize_t recvfrom(int socket, void *buffer, size_t length, - int flags, struct sockaddr *address, socklen_t *address_len) -{ - try { - auto& fd = FD_map::_get(socket); - return fd.recvfrom(buffer, length, flags, address, address_len); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} -int listen(int socket, int backlog) -{ - PRINT("listen(%d, %d)\n", socket, backlog); - try { - auto& fd = FD_map::_get(socket); - return fd.listen(backlog); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} -int accept(int socket, struct sockaddr *address, socklen_t *len) -{ - PRINT("accept(%d, %p, %d)\n", socket, address, *len); - try { - auto& fd = FD_map::_get(socket); - return fd.accept(address, len); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} -int bind(int socket, const struct sockaddr* address, socklen_t len) -{ - PRINT("bind(%d, %p, %d)\n", socket, address, len); - try { - auto& fd = FD_map::_get(socket); - return fd.bind(address, len); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} -int getsockopt(int socket, int level, int option_name, - void *option_value, socklen_t *option_len) -{ - PRINT("getsockopt(%d, %d, %d, %p, %d)\n", socket, level, - option_name, option_value, option_len); - try { - auto& fd = FD_map::_get(socket); - return fd.getsockopt(level, option_name, option_value, option_len); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} -int setsockopt(int socket, int level, int option_name, - const void *option_value, socklen_t option_len) -{ - PRINT("setsockopt(%d, %d, %d, %p, %d)\n", socket, level, - option_name, option_value, option_len); - try { - auto& fd = FD_map::_get(socket); - return fd.setsockopt(level, option_name, option_value, option_len); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} -int shutdown(int socket, int how) -{ - PRINT("shutdown(%d, %d)\n", socket, how); - try { - auto& fd = FD_map::_get(socket); - return fd.shutdown(how); - } catch (const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -bool validate_sockaddr_in(const struct sockaddr* address, socklen_t len) -{ - // The specified address is not a valid address for the address family of the specified socket. - if (UNLIKELY(len < sizeof(struct sockaddr_in))) { - PRINT("sockaddr_in length too short: %u vs %lu\n", len, sizeof(struct sockaddr_in)); - return false; - } - - return true; -} diff --git a/src/posix/sys/stat.cpp b/src/posix/sys/stat.cpp deleted file mode 100644 index 5891150254..0000000000 --- a/src/posix/sys/stat.cpp +++ /dev/null @@ -1,307 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef DEFAULT_UMASK -#define DEFAULT_UMASK 002 -#endif - -extern fs::Disk_ptr& fs_disk(); -inline unsigned round_up(unsigned n, unsigned div) { - Expects(div > 0); - return (n + div - 1) / div; -} - -int chmod(const char *path, mode_t mode) { - (void) path; - (void) mode; - errno = EROFS; - return -1; -} - -int fchmod(int fildes, mode_t mode) -{ - try { - auto& fd = FD_map::_get(fildes); - return fd.fchmod(mode); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -int fchmodat(int filedes, const char *path, mode_t mode, int flag) -{ - try { - auto& fd = FD_map::_get(filedes); - return fd.fchmodat(path, mode, flag); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -int fstatat(int filedes, const char *path, struct stat *buf, int flag) -{ - if (filedes == AT_FDCWD) - { - char cwd_buf[PATH_MAX]; - char abs_path[PATH_MAX]; - if (getcwd(cwd_buf, PATH_MAX)) { - snprintf(abs_path, PATH_MAX, "%s/%s", cwd_buf, path); - } - return stat(abs_path, buf); - } - else - { - try { - auto& fd = FD_map::_get(filedes); - return fd.fstatat(path, buf, flag); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } - } -} - -int futimens(int filedes, const struct timespec times[2]) -{ - try { - auto& fd = FD_map::_get(filedes); - return fd.futimens(times); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -int utimensat(int filedes, const char *path, const struct timespec times[2], int flag) -{ - try { - auto& fd = FD_map::_get(filedes); - return fd.utimensat(path, times, flag); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -int mkdir(const char *path, mode_t mode) -{ - (void) path; - (void) mode; - errno = EROFS; - return -1; -} - -int mkdirat(int filedes, const char *path, mode_t mode) -{ - try { - auto& fd = FD_map::_get(filedes); - return fd.mkdirat(path, mode); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -int mkfifo(const char *path, mode_t mode) -{ - (void) path; - (void) mode; - errno = EROFS; - return -1; -} - -int mkfifoat(int filedes, const char *path, mode_t mode) -{ - try { - auto& fd = FD_map::_get(filedes); - return fd.mkfifoat(path, mode); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -int mknod(const char *path, mode_t mode, dev_t dev) -{ - (void) path; - (void) mode; - (void) dev; - errno = EROFS; - return -1; -} - -int mknodat(int filedes, const char *path, mode_t mode, dev_t dev) -{ - try { - auto& fd = FD_map::_get(filedes); - return fd.mknodat(path, mode, dev); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -int stat(const char *path, struct stat *buf) -{ - if (buf == nullptr) - { - PRINT("stat(%s, %p) == %d\n", path, buf, -1); - errno = EFAULT; - return -1; - } - memset(buf, 0, sizeof(struct stat)); - try { - auto ent = fs::VFS::stat_sync(path); - if (ent.is_valid()) - { - if (ent.is_file()) buf->st_mode = S_IFREG; - if (ent.is_dir()) buf->st_mode = S_IFDIR; - buf->st_dev = ent.device_id(); - buf->st_ino = ent.block(); - buf->st_nlink = 1; - buf->st_size = ent.size(); - buf->st_atime = ent.modified(); - buf->st_ctime = ent.modified(); - buf->st_mtime = ent.modified(); - buf->st_blocks = buf->st_size > 0 ? round_up(buf->st_size, 512) : 0; - buf->st_blksize = fs::MemDisk::SECTOR_SIZE; - PRINT("stat(%s, %p) == %d\n", path, buf, 0); - return 0; - } - else { - PRINT("stat(%s, %p) == %d\n", path, buf, -1); - errno = EIO; - return -1; - } - } - catch (...) - { - PRINT("stat(%s, %p) == %d\n", path, buf, -1); - errno = EIO; - return -1; - } -} -int lstat(const char *path, struct stat *buf) -{ - // NOTE: should stat symlinks, instead of following them - return stat(path, buf); -} - -mode_t umask(mode_t cmask) -{ - (void) cmask; - return DEFAULT_UMASK; -} - -int fstat(int fd, struct stat* stat_buf) -{ - if (fd < 0) { - PRINT("fstat(%d, %p) == %d\n", fd, stat_buf, -1); - errno = EBADF; - return -1; - } - if (stat_buf == nullptr) { - PRINT("fstat(%d, %p) == %d\n", fd, stat_buf, -1); - errno = EINVAL; - return -1; - } - if (fd < 4 || fd == 4) - { - if (fd == 4) { - PRINT("fstat(%d, %p) == %d\n", fd, stat_buf, 0); - } - stat_buf->st_dev = 6; - stat_buf->st_ino = fd; - stat_buf->st_mode = 0x21b6; - stat_buf->st_nlink = 1; - stat_buf->st_uid = 0; - stat_buf->st_gid = 0; - stat_buf->st_rdev = 265; - stat_buf->st_size = 0; - if (fd == 4) - { - stat_buf->st_atime = RTC::now(); - stat_buf->st_mtime = RTC::now(); - stat_buf->st_ctime = RTC::now(); - } - stat_buf->st_blksize = 4096; - stat_buf->st_blocks = 0; - return 0; - } - PRINT("fstat(%d, %p) == %d\n", fd, stat_buf, -1); - errno = ENOSYS; - return -1; -} - -extern "C" -int __xstat(int ver, const char * path, struct stat * stat_buf) -{ - PRINT("__xstat(%d, '%s', %p)\n", ver, path, stat_buf); - errno = ENOSYS; - return -1; -} - -extern "C" -int __lxstat(int ver, const char * path, struct stat * stat_buf) -{ - PRINT("__lxstat(%d, '%s', %p)\n", ver, path, stat_buf); - errno = ENOSYS; - return -1; -} - -#define LINUX_STAT_VER 1 - -extern "C" -int __fxstat(int ver, int fd, struct stat * stat_buf) -{ - PRINT("__fxstat(%d, %d, %p)\n", ver, fd, stat_buf); - if (fd < 0) { - errno = EBADF; - return -1; - } - if (stat_buf == nullptr) { - errno = EINVAL; - return -1; - } - if (ver != LINUX_STAT_VER) { - errno = EINVAL; - return -1; - } - - return fstat(fd, stat_buf); -} diff --git a/src/posix/sys/utsname.cpp b/src/posix/sys/utsname.cpp deleted file mode 100644 index 49b345c699..0000000000 --- a/src/posix/sys/utsname.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -int uname(struct utsname *name) { - - strcpy(name->sysname, "IncludeOS"); - - /* sprintf(name->nodename, "Node %d\n", _Objects_Local_node); */ - strcpy(name->nodename, "IncludeOS-node"); - // same as hostname - - strcpy(name->release, OS::version().c_str()); - - strcpy(name->version, OS::version().c_str()); - - /* sprintf(name->machine, "%s/%s", CPU_NAME, CPU_MODEL_NAME); */ - // strcpy(name->machine, OS::machine().c_str()); - strcpy(name->machine, "x86_64"); - - return 0; -} diff --git a/src/posix/syslog.cpp b/src/posix/syslog.cpp deleted file mode 100644 index a6a0f08027..0000000000 --- a/src/posix/syslog.cpp +++ /dev/null @@ -1,144 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -static const char* ident_ = nullptr; /* what is prepended to each message - default could be the name of the program */ - -static int logopt_ = 0; /* default logging option */ -static int facility_ = LOG_USER; /* default facility */ - -static int log_mask = 0xff; /* mask of priorities to be logged - default: allows all priorities to be logged */ - -void closelog(void) { - // Back to default values: - ident_ = nullptr; - logopt_ = 0; - facility_ = LOG_USER; - Syslog::closelog(); -} - -/* - The logopt argument indicates logging options. Values for logopt are constructed - by a bitwise-inclusive OR of zero or more of the following: look above. - - Options (logopt) is a bit string, with the bits as defined by the single bit masks - LOG_PERROR - LOG_CONS - LOG_PID - LOG_NDELAY - LOG_ODELAY - LOG_NOWAIT - If any other bit in options is on, the result is undefined -*/ -/* - The following example causes subsequent calls to syslog() to log the process ID with each message, - and to write messages to the system console if they cannot be sent to the logging facility. - - #include - char *ident = "Process demo"; - int logopt = LOG_PID | LOG_CONS; - int facility = LOG_USER; - ... - openlog(ident, logopt, facility); -*/ -void openlog(const char* ident, int logopt, int facility) { - if (ident != nullptr) - ident_ = ident; - - logopt_ = logopt; - - facility_ = facility; - - Syslog::openlog(ident_, logopt_, facility_); -} - -/* - Shall set the log priority mask for the current process to maskpri and - return the previous mask. - If the maskpri argument is 0, the current log mask is not modified. - Calls by the current process to syslog() with a priority not set in maskpri - shall be rejected. - - The default log mask allows all priorities to be logged. - (on top set to 0xff) - - A call to openlog() is not required prior to calling setlogmask(). - - The function shall return the previous log priority mask. - - Using setlogmask(): - The following example causes subsequent calls to syslog() to accept error messages, and to reject all other messages. - #include - int result; - int mask = LOG_MASK (LOG_ERR); - ... - result = setlogmask(mask); -*/ -int setlogmask(int maskpri) { - int old_mask = log_mask; - - if (maskpri != 0) // the mask has been modified - log_mask = maskpri; - - return old_mask; -} - -/* - Shall send a message to an implementation-defined logging facility, - which may log it in an implementation-defined system log, write it to the - system console, forward it to a list of users, or forward it to the logging - facility on another host over the network. The logged message shall include - a message header and a message body. The message header contains at least a - timestamp and a tag string. - - The message body is generated from the message (argument) and following arguments - in the same manner as if these were arguments to printf(), except that the additional - conversion specification %m shall be recognized; it shall convert no arguments, - shall cause the output of the error message string associated with the value of - errno on entry to syslog(), and may be mixed with argument specifications of the - "%n$" form. If a complete conversion specification with the m conversion specifier - character is not just %m, the behavior is undefined. A trailing may be - added if needed. - - Values of the priority argument are formed by OR'ing together a severity-level value - and an optional facility value. If no facility value is specified, the current - default facility value is used. - Possible values of severity: look above. - Possible facility values: look above. -*/ -/* - Calls by the current process to syslog() with a priority not set in maskpri shall be rejected. -*/ -/* - You don't have to use openlog. If you call syslog without having called openlog, syslog - just opens the connection implicitly and uses defaults for the information in ident and - options -*/ -void syslog(int priority, const char* message, ... /* arguments*/) { - /* Arguments */ - va_list ap; - va_start(ap, message); - - // Calling syslog-function that takes a va_list (uses vsnprintf) - Syslog::syslog(priority, message, ap); - - va_end(ap); -} diff --git a/src/posix/tcp_fd.cpp b/src/posix/tcp_fd.cpp deleted file mode 100644 index 7c8d38ba5e..0000000000 --- a/src/posix/tcp_fd.cpp +++ /dev/null @@ -1,384 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include - -#define POSIX_STRACE -#ifdef POSIX_STRACE -#define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) -#else -#define PRINT(fmt, ...) /* fmt */ -#endif - -using namespace net; - -// return the "currently selected" networking stack -static auto& net_stack() { - return Inet4::stack(); -} - -int TCP_FD::read(void* data, size_t len) -{ - return recv(data, len, 0); -} -int TCP_FD::write(const void* data, size_t len) -{ - return send(data, len, 0); -} - -int TCP_FD::close() -{ - // connection - if (this->cd) { - PRINT("TCP: close(%s)\n", cd->to_string().c_str()); - int ret = cd->close(); - delete cd; cd = nullptr; - return ret; - } - // listener - if (this->ld) { - PRINT("TCP: close(%s)\n", ld->to_string().c_str()); - int ret = ld->close(); - delete ld; ld = nullptr; - return ret; - } - errno = EBADF; - return -1; -} - -int TCP_FD::connect(const struct sockaddr* address, socklen_t address_len) -{ - if (is_listener()) { - PRINT("TCP: connect(%s) failed\n", ld->to_string().c_str()); - // already listening on port - errno = EINVAL; - return -1; - } - if (this->cd) { - PRINT("TCP: connect(%s) failed\n", cd->to_string().c_str()); - // if its straight-up connected, return that - if (cd->conn->is_connected()) { - errno = EISCONN; - return -1; - } - // if the connection isn't closed, we can just assume its being used already - if (!cd->conn->is_closed()) { - errno = EALREADY; - return -1; - } - } - - if (address_len != sizeof(sockaddr_in)) { - errno = EINVAL; // checkme? - return -1; - } - auto* inaddr = (sockaddr_in*) address; - - auto addr = ip4::Addr(inaddr->sin_addr.s_addr); - auto port = ::htons(inaddr->sin_port); - - PRINT("TCP: connect(%s:%u)\n", addr.to_string().c_str(), port); - - auto outgoing = net_stack().tcp().connect({addr, port}); - // O_NONBLOCK is set for the file descriptor for the socket and the connection - // cannot be immediately established; the connection shall be established asynchronously. - if (this->is_blocking() == false) { - errno = EINPROGRESS; - return -1; - } - - // wait for connection state to change - while (not (outgoing->is_connected() || outgoing->is_closing() || outgoing->is_closed())) { - OS::block(); - } - // set connection whether good or bad - if (outgoing->is_connected()) { - // out with the old - delete this->cd; - // in with the new - this->cd = new TCP_FD_Conn(outgoing); - cd->set_default_read(); - return 0; - } - // failed to connect - // TODO: try to distinguish the reason for connection failure - errno = ECONNREFUSED; - return -1; -} - - -ssize_t TCP_FD::send(const void* data, size_t len, int fmt) -{ - if (!cd) { - errno = EINVAL; - return -1; - } - return cd->send(data, len, fmt); -} -ssize_t TCP_FD::recv(void* dest, size_t len, int flags) -{ - if (!cd) { - errno = EINVAL; - return -1; - } - return cd->recv(dest, len, flags); -} - -int TCP_FD::accept(struct sockaddr *__restrict__ addr, socklen_t *__restrict__ addr_len) -{ - if (!ld) { - errno = EINVAL; - return -1; - } - return ld->accept(addr, addr_len); -} -int TCP_FD::listen(int backlog) -{ - if (!ld) { - errno = EINVAL; - return -1; - } - return ld->listen(backlog); -} -int TCP_FD::bind(const struct sockaddr *addr, socklen_t addrlen) -{ - // - if (cd) { - errno = EINVAL; - return -1; - } - // verify socket address - if (addrlen != sizeof(sockaddr_in)) { - errno = EINVAL; - return -1; - } - auto* sin = (sockaddr_in*) addr; - // verify its AF_INET - if (sin->sin_family != AF_INET) { - errno = EAFNOSUPPORT; - return -1; - } - // use sin_port for bind - // its network order ... so swap that shit: - uint16_t port = ::htons(sin->sin_port); - // ignore IP address (FIXME?) - /// TODO: verify that the IP is "local" - try { - auto& L = net_stack().tcp().listen(port); - // remove existing listener - if (ld) { - int ret = ld->close(); - if (ret < 0) return ret; - delete ld; - } - // create new one - ld = new TCP_FD_Listen(L); - return 0; - - } catch (...) { - errno = EADDRINUSE; - return -1; - } -} -int TCP_FD::shutdown(int mode) -{ - if (!cd) { - errno = EINVAL; - return -1; - } - return cd->shutdown(mode); -} - -/// socket default handler getters - -TCP_FD::on_read_func TCP_FD::get_default_read_func() -{ - if (cd) { - return {cd, &TCP_FD_Conn::recv_to_ringbuffer}; - } - if (ld) { - return - [this] (net::tcp::buffer_t) { - // what to do here? - }; - } - throw std::runtime_error("Invalid socket"); -} -TCP_FD::on_write_func TCP_FD::get_default_write_func() -{ - return [] {}; -} -TCP_FD::on_except_func TCP_FD::get_default_except_func() -{ - return [] {}; -} - -/// socket as connection - -void TCP_FD_Conn::recv_to_ringbuffer(net::tcp::buffer_t buffer) -{ - if (readq.free_space() < (ssize_t) buffer->size()) { - // make room for data - int needed = buffer->size() - readq.free_space(); - int discarded = readq.discard(needed); - assert(discarded == needed); - } - // add data to ringbuffer - int written = readq.write(buffer->data(), buffer->size()); - assert(written == (ssize_t) buffer->size()); -} -void TCP_FD_Conn::set_default_read() -{ - // readq buffering (4kb at a time) - conn->on_read(4096, {this, &TCP_FD_Conn::recv_to_ringbuffer}); -} -ssize_t TCP_FD_Conn::send(const void* data, size_t len, int) -{ - if (not conn->is_connected()) { - errno = ENOTCONN; - return -1; - } - - bool written = false; - conn->on_write([&written] (bool) { written = true; }); // temp - - conn->write(data, len); - - // sometimes we can just write and forget - if (written) return len; - while (!written) { - OS::block(); - } - - conn->on_write(nullptr); // temp - return len; -} -ssize_t TCP_FD_Conn::recv(void* dest, size_t len, int) -{ - // if the connection is closed or closing: read returns 0 - if (conn->is_closed() || conn->is_closing()) return 0; - if (not conn->is_connected()) { - errno = ENOTCONN; - return -1; - } - // read some bytes from readq - int bytes = readq.read((char*) dest, len); - if (bytes) return bytes; - - bool done = false; - bytes = 0; - - // block and wait for more - conn->on_read(len, - [&done, &bytes, dest] (auto buffer) { - // copy the data itself - if (buffer->size() > 0) - memcpy(dest, buffer->data(), buffer->size()); - // we are done - done = true; - bytes = buffer->size(); - }); - - // BLOCK HERE - while (!done || !conn->is_readable()) { - OS::block(); - } - // restore - this->set_default_read(); - return bytes; -} -int TCP_FD_Conn::close() -{ - conn->close(); - // wait for connection to close completely - while (!conn->is_closed()) { - OS::block(); - } - return 0; -} -int TCP_FD_Conn::shutdown(int mode) -{ - if (not conn->is_connected()) { - errno = ENOTCONN; - return -1; - } - switch (mode) { - case SHUT_RDWR: - conn->close(); - return 0; - case SHUT_RD: - printf("Ignoring shutdown(SHUT_RD)\n"); - return 0; - case SHUT_WR: - printf("Ignoring shutdown(SHUT_WR)\n"); - return 0; - default: - errno = EINVAL; - return -1; - } -} - -/// socket as listener - -int TCP_FD_Listen::listen(int backlog) -{ - listener.on_connect( - [this, backlog] (net::tcp::Connection_ptr conn) - { - // remove oldest if full - if ((int) this->connq.size() >= backlog) - this->connq.pop_back(); - // new connection - this->connq.push_front(conn); - /// if someone is blocking they should be leaving right about now - }); - return 0; -} -int TCP_FD_Listen::accept(struct sockaddr *__restrict__ addr, socklen_t *__restrict__ addr_len) -{ - // block until connection appears - while (connq.empty()) { - OS::block(); - } - // retrieve connection from queue - auto sock = connq.back(); - connq.pop_back(); - // make sure socket is connected, as promised - if (not sock->is_connected()) { - errno = ENOTCONN; - return -1; - } - // create connected TCP socket - auto& fd = FD_map::_open(); - fd.cd = new TCP_FD_Conn(sock); - // set address and length - auto* sin = (sockaddr_in*) addr; - sin->sin_family = AF_INET; - sin->sin_port = sock->remote().port(); - sin->sin_addr.s_addr = sock->remote().address().whole; - *addr_len = sizeof(sockaddr_in); - // return socket - return fd.get_id(); -} -int TCP_FD_Listen::close() -{ - listener.close(); - return 0; -} diff --git a/src/posix/termios.cpp b/src/posix/termios.cpp deleted file mode 100644 index 2d96c6f0db..0000000000 --- a/src/posix/termios.cpp +++ /dev/null @@ -1,27 +0,0 @@ -//#include -#include -#include - -typedef unsigned long tcflag_t; -typedef unsigned long cc_t; -#define NCCS 64 - -struct termios -{ - tcflag_t c_iflag; // Input modes. - tcflag_t c_oflag; // Output modes. - tcflag_t c_cflag; // Control modes. - tcflag_t c_lflag; // Local modes. - cc_t c_cc[NCCS]; // Control characters. -}; - -extern "C" -int tcsetattr(int fildes, int optional_actions, const struct termios *termios_p) -{ - printf("tcsetattr(%d, %d, %p)\n", fildes, optional_actions, termios_p); -} -extern "C" -int tcgetattr(int fildes, struct termios *termios_p) -{ - printf("tcgetattr(%d, %p)\n", fildes, termios_p); -} diff --git a/src/posix/ucontext.cpp b/src/posix/ucontext.cpp deleted file mode 100644 index 7fed6df99a..0000000000 --- a/src/posix/ucontext.cpp +++ /dev/null @@ -1,103 +0,0 @@ -#include -#include -#include -#include - -// default successor context -static ucontext_t default_successor_context; - -extern "C" void restore_context_stack(); - -__attribute__((constructor)) -void initialize_default_successor_context() -{ - static const int DEFAULT_SIZE = 1024; - default_successor_context.uc_link = nullptr; - // create a stack for the context - auto* stack = new char[DEFAULT_SIZE]; - default_successor_context.uc_stack.ss_sp = stack + DEFAULT_SIZE; - default_successor_context.uc_stack.ss_size = DEFAULT_SIZE; - makecontext(&default_successor_context, - [](){ Service::stop(); }, - 0); -} - -static void prepare_context_stack(ucontext_t *ucp, ucontext_t *successor_context, int argc, va_list args) -{ - size_t* stack_ptr = (size_t *)(ucp->uc_mcontext.ret_esp); - - // successor_context - stack_ptr -= 1; - - if(successor_context == nullptr) { - stack_ptr[0] = (size_t)&default_successor_context; - } - else { - stack_ptr[0] = (size_t)successor_context; - } - - // arguments - stack_ptr -= (argc + 1); - stack_ptr[0] = (size_t)argc; - - for(int i = 1; i <= argc; i++) { - stack_ptr[i] = va_arg(args, size_t); - } - - //return function - stack_ptr -= 1; - stack_ptr[0] = (size_t)restore_context_stack; - - ucp->uc_mcontext.ret_esp = (size_t)stack_ptr; -} - -extern "C" { - -void makecontext(ucontext_t *ucp, void (*func)(), int argc, ...) -{ - if (ucp == nullptr || func == nullptr || argc < 0) { - errno = EINVAL; - PRINT("makecontext(%p, %p, %d, ...) = -1\n", ucp, func, argc); - return; - } - - if (ucp->uc_stack.ss_sp == nullptr || ucp->uc_stack.ss_size == 0 - || ucp->uc_stack.ss_size > ucontext_t::MAX_STACK_SIZE) { - - errno = EINVAL; - PRINT("makecontext(%p, %p, %d, ...) = -1\n", ucp, func, argc); - return; - } - - ucp->uc_mcontext.ret_esp = (size_t) ucp->uc_stack.ss_sp; - ucp->uc_mcontext.ebp = ucp->uc_mcontext.ret_esp; - ucp->uc_mcontext.eip = (size_t) func; - - va_list args; - va_start(args, argc); - - prepare_context_stack(ucp, ucp->uc_link, argc, args); - - va_end(args); - PRINT("makecontext(%p, %p, %d, ...) = 0\n", ucp, func, argc); -} - -int swapcontext(ucontext_t *oucp, ucontext_t *ucp) -{ - if(oucp == nullptr || ucp == nullptr) { - errno = EINVAL; - return -1; - } - - volatile bool swapped = false; - int result = getcontext(oucp); - - if(result != -1 && !swapped){ - swapped = true; - return setcontext(ucp); - } - - return result; -} - -} diff --git a/src/posix/ucontext_asm.asm b/src/posix/ucontext_asm.asm deleted file mode 100644 index 9b765adee1..0000000000 --- a/src/posix/ucontext_asm.asm +++ /dev/null @@ -1,163 +0,0 @@ -USE32 -extern errno - -global getcontext -global setcontext -global restore_context_stack - -%define EINVAL (22) -%define NULL (0) -%define sizeof_size_t (4) - -struc ucontext_t - .uc_link: resd 1 - .uc_stack: resb 12 - .uc_mcontext: resb 40 -endstruc - -struc stack_t - .ss_sp: resd 1 - .ss_size: resd 1 - .ss_flags: resd 1 -endstruc - -struc mcontext_t - .edi: resd 1 - .esi: resd 1 - .ebp: resd 1 - .ebx: resd 1 - .edx: resd 1 - .ecx: resd 1 - .eax: resd 1 - - .floating_point_env: resb 28 - .eip: resd 1 - .flags: resd 1 - .ret_esp: resd 1 -endstruc - -section .text - -;; ============================================================================= -;; Save the calling context into ucontext_t -;; -;; @param [ESP + 4] Pointer to a mcontext_t struct. -;; -;; @return -1 if an error occured, 0 for success. -;; ============================================================================= -getcontext: - mov eax, DWORD [esp + 4] - cmp eax, NULL - jne .valid_arg - mov eax, -1 - ret - -.valid_arg: - add eax, ucontext_t.uc_mcontext - - ;;General purpose registers - mov DWORD [eax + mcontext_t.edi], edi - mov DWORD [eax + mcontext_t.esi], esi - mov DWORD [eax + mcontext_t.ecx], ecx - mov DWORD [eax + mcontext_t.edx], edx - mov DWORD [eax + mcontext_t.ebp], ebp - mov DWORD [eax + mcontext_t.ebx], ebp - - ;; Floating point env - fnstenv [eax + mcontext_t.floating_point_env] - fldenv [eax + mcontext_t.floating_point_env] - - ;;Eip - mov ecx, [esp] - mov DWORD [eax + mcontext_t.eip], ecx - - ;;Return stack - lea ecx, [esp + 4] - mov DWORD [eax + mcontext_t.ret_esp], ecx - - ;;Flags - pushfd - mov ebx, [esp] - mov DWORD [eax + mcontext_t.flags], ebx - popfd - - sub eax, 12 - mov [eax + stack_t.ss_sp], ecx - - ;;Size doesn't matter ( ͡° ͜ʖ ͡°) - mov DWORD [eax + stack_t.ss_size], 4096 - - mov eax, 0 - -.return: - ret - -;; ============================================================================= -;; Switch to other context -;; -;; @param [ESP + 4] Pointer to a mcontext_t struct. -;; -;; @return -1 if an error occured, doesn't return if successful. -;; ============================================================================= -setcontext: - mov eax, DWORD [esp + 4] - test eax, eax - - jne .valid_arg - - mov eax, -1 - mov DWORD [errno], EINVAL - ret - -.valid_arg: - mov eax, [esp + 4] - add eax, 16 - mov edi, [eax + mcontext_t.edi] - mov esi, [eax + mcontext_t.esi] - mov ebp, [eax + mcontext_t.ebp] - mov ebx, [eax + mcontext_t.ebx] - mov edx, [eax + mcontext_t.edx] - mov ecx, [eax + mcontext_t.ecx] - - fldenv [eax + mcontext_t.floating_point_env] - - push DWORD [eax + mcontext_t.flags] - popfd - - ;; New stack - push DWORD [eax + mcontext_t.ret_esp] - pop esp - - ;; Eip - push DWORD [eax + mcontext_t.eip] - -.return: - ret - -;; ============================================================================= -;; When a context finishes its execution, it will return to this function. -;; This function will prepare the stack for 'setcontext', to successfuly -;; return to the successor context. -;; -;; @param [ESP + 4] Returning context's argc -;; @param [ESP + 8] - [ESP + 8 + (4 * argc)] Returning context's arguments -;; -;; @return None -;; ============================================================================= - -restore_context_stack: - ;; stack pointer - lea ebx, [esp] - - ;;offset to the successor context pointer - mov eax, sizeof_size_t - mov ecx, DWORD [esp] - mul ecx - add ebx, eax - add ebx, 4 - - ;; successor context pointer - mov eax, [ebx] - mov [esp + 4], eax - ;;Shouldn't return - jmp setcontext diff --git a/src/posix/udp_fd.cpp b/src/posix/udp_fd.cpp deleted file mode 100644 index 47eaf11537..0000000000 --- a/src/posix/udp_fd.cpp +++ /dev/null @@ -1,389 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include // OS::block() -#include - -#define POSIX_STRACE -#ifdef POSIX_STRACE -#define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) -#else -#define PRINT(fmt, ...) /* fmt */ -#endif - -// return the "currently selected" networking stack -static net::Inet& net_stack() { - return net::Inet4::stack<> (); -} - -size_t UDP_FD::max_buffer_msgs() const -{ - return (rcvbuf_ / net_stack().udp().max_datagram_size()); -} - -void UDP_FD::recv_to_buffer(net::UDPSocket::addr_t addr, - net::UDPSocket::port_t port, const char* buf, size_t len) -{ - // only recv to buffer if not full - if(buffer_.size() < max_buffer_msgs()) { - // copy data into to-be Message buffer - auto buff = net::tcp::buffer_t(new std::vector (buf, buf + len)); - // emplace the message in buffer - buffer_.emplace_back(htonl(addr.whole), htons(port), std::move(buff)); - } -} - -int UDP_FD::read_from_buffer(void* buffer, size_t len, int flags, - struct sockaddr* address, socklen_t* address_len) -{ - assert(!buffer_.empty() && "Trying to read from empty buffer"); - - auto& msg = buffer_.front(); - auto& mbuf = msg.buffer; - - int bytes = std::min(len, mbuf->size()); - memcpy(buffer, mbuf->data(), bytes); - - if(address != nullptr) { - memcpy(address, &msg.src, std::min(*address_len, (uint32_t) sizeof(struct sockaddr_in))); - *address_len = sizeof(struct sockaddr_in); - } - - if(!(flags & MSG_PEEK)) - buffer_.pop_front(); - - return bytes; -} -void UDP_FD::set_default_recv() -{ - assert(this->sock != nullptr && "Default recv called on nullptr"); - this->sock->on_read({this, &UDP_FD::recv_to_buffer}); -} -UDP_FD::~UDP_FD() -{ - // shutdown underlying socket, makes sure no callbacks are called on dead fd - if(this->sock) - sock->close(); -} -int UDP_FD::read(void* buffer, size_t len) -{ - return recv(buffer, len, 0); -} -int UDP_FD::write(const void* buffer, size_t len) -{ - return send(buffer, len, 0); -} -int UDP_FD::close() -{ - return -1; -} -int UDP_FD::bind(const struct sockaddr* address, socklen_t len) -{ - // we can assume this has already been bound since there is a pointer - if(UNLIKELY(this->sock != nullptr)) { - errno = EINVAL; - return -1; - } - // invalid address - if(UNLIKELY(len != sizeof(struct sockaddr_in))) { - errno = EINVAL; - return -1; - } - // Bind - const auto port = ((sockaddr_in*)address)->sin_port; - auto& udp = net_stack().udp(); - try - { - this->sock = (port) ? &udp.bind(ntohs(port)) : &udp.bind(); - set_default_recv(); - PRINT("UDP: bind(%s)\n", sock->local().to_string().c_str()); - return 0; - } - catch(const net::UDP::Port_in_use_exception&) - { - errno = EADDRINUSE; - return -1; - } -} -int UDP_FD::connect(const struct sockaddr* address, socklen_t address_len) -{ - if (UNLIKELY(!validate_sockaddr_in(address, address_len))) { - errno = EINVAL; - return -1; - } - - // If the socket has not already been bound to a local address, - // connect() shall bind it to an address which is an unused local address. - if(this->sock == nullptr) { - this->sock = &net_stack().udp().bind(); - set_default_recv(); - } - - const auto& addr = *((sockaddr_in*)address); - if (addr.sin_family == AF_UNSPEC) - { - peer_.sin_addr.s_addr = 0; - peer_.sin_port = 0; - } - else - { - peer_.sin_family = AF_INET; - peer_.sin_addr = addr.sin_addr; - peer_.sin_port = addr.sin_port; - } - PRINT("UDP: connect(%s:%u)\n", - net::IP4::addr(peer_.sin_addr.s_addr).to_string().c_str(), - htons(peer_.sin_port)); - - return 0; -} -ssize_t UDP_FD::send(const void* message, size_t len, int flags) -{ - if(!is_connected()) { - errno = EDESTADDRREQ; - return -1; - } - PRINT("UDP: send(%lu, %x)\n", len, flags); - - return sendto(message, len, flags, (struct sockaddr*)&peer_, sizeof(peer_)); -} - -ssize_t UDP_FD::sendto(const void* message, size_t len, int, - const struct sockaddr* dest_addr, socklen_t dest_len) -{ - // The specified address is not a valid address for the address family of the specified socket. - if(UNLIKELY(dest_len != sizeof(struct sockaddr_in))) { - errno = EINVAL; - return -1; - } - // Bind a socket if we dont already have one - if(this->sock == nullptr) { - this->sock = &net_stack().udp().bind(); - set_default_recv(); - } - const auto& dest = *((sockaddr_in*)dest_addr); - // If the socket protocol supports broadcast and the specified address - // is a broadcast address for the socket protocol, - // sendto() shall fail if the SO_BROADCAST option is not set for the socket. - if(!broadcast_ && dest.sin_addr.s_addr == INADDR_BROADCAST) { // Fix me - return -1; - } - - // Sending - bool written = false; - this->sock->sendto(ntohl(dest.sin_addr.s_addr), ntohs(dest.sin_port), message, len, - [&written]() { written = true; }); - - while(!written) - OS::block(); - - return len; -} -ssize_t UDP_FD::recv(void* buffer, size_t len, int flags) -{ - PRINT("UDP: recv(%lu, %x)\n", len, flags); - return recvfrom(buffer, len, flags, nullptr, 0); -} -ssize_t UDP_FD::recvfrom(void *__restrict__ buffer, size_t len, int flags, - struct sockaddr *__restrict__ address, socklen_t *__restrict__ address_len) -{ - if(UNLIKELY(this->sock == nullptr)) { - errno = EINVAL; // - return -1; - } - - // Read from buffer if not empty - if(!buffer_.empty()) - { - return read_from_buffer(buffer, len, flags, address, address_len); - } - // Else make a blocking receive - else - { - int bytes = 0; - bool done = false; - - this->sock->on_read(net::UDPSocket::recvfrom_handler::make_packed( - [&bytes, &done, this, - buffer, len, flags, address, address_len] - (net::UDPSocket::addr_t addr, net::UDPSocket::port_t port, - const char* data, size_t data_len) - { - // if this already been called once while blocking, buffer - if(done) { - recv_to_buffer(addr, port, data, data_len); - return; - } - - bytes = std::min(len, data_len); - memcpy(buffer, data, bytes); - - // TODO: If the actual length of the address is greater than the length of the supplied sockaddr structure, - // the stored address shall be truncated. - if(address != nullptr) { - auto& sender = *((sockaddr_in*)address); - sender.sin_family = AF_INET; - sender.sin_port = htons(port); - sender.sin_addr.s_addr = htonl(addr.whole); - *address_len = sizeof(struct sockaddr_in); - } - done = true; - - // Store in buffer if PEEK - if(flags & MSG_PEEK) - recv_to_buffer(addr, port, data, data_len); - })); - - // Block until (any) data is read - while(!done) - OS::block(); - - set_default_recv(); - - return bytes; - } -} -int UDP_FD::getsockopt(int level, int option_name, - void *option_value, socklen_t *option_len) -{ - PRINT("UDP: getsockopt(%d, %d)\n", level, option_name); - if(level != SOL_SOCKET) - return -1; - - switch(option_name) - { - case SO_ACCEPTCONN: - { - errno = ENOPROTOOPT; - return -1; - } - case SO_BROADCAST: - { - if(*option_len < (int)sizeof(int)) - { - errno = EINVAL; - return -1; - } - - *((int*)option_value) = broadcast_; - *option_len = sizeof(broadcast_); - return 0; - } - case SO_KEEPALIVE: - { - errno = ENOPROTOOPT; - return -1; - } - case SO_RCVBUF: - { - if(*option_len < (int)sizeof(int)) - { - errno = EINVAL; - return -1; - } - - *((int*)option_value) = rcvbuf_; - *option_len = sizeof(rcvbuf_); - return 0; - } - // Address can always be reused in IncludeOS - case SO_REUSEADDR: - { - if(*option_len < (int)sizeof(int)) - { - errno = EINVAL; - return -1; - } - - *((int*)option_value) = 1; - *option_len = sizeof(int); - return 0; - } - case SO_TYPE: - { - if(*option_len < (int)sizeof(int)) - { - errno = EINVAL; - return -1; - } - - *((int*)option_value) = SOCK_DGRAM; - *option_len = sizeof(int); - return 0; - } - - default: - return -1; - } // < switch(option_name) -} - -int UDP_FD::setsockopt(int level, int option_name, - const void *option_value, socklen_t option_len) -{ - PRINT("UDP: setsockopt(%d, %d, ... %d)\n", level, option_name, option_len); - if(level != SOL_SOCKET) - return -1; - - switch(option_name) - { - case SO_BROADCAST: - { - if(option_len < (int)sizeof(int)) - return -1; - - broadcast_ = *((int*)option_value); - return 0; - } - case SO_KEEPALIVE: - { - errno = ENOPROTOOPT; - return -1; - } - case SO_RCVBUF: - { - if(option_len < (int)sizeof(int)) - return -1; - - rcvbuf_ = *((int*)option_value); - return 0; - } - // Address can always be reused in IncludeOS - case SO_REUSEADDR: - { - return 0; - } - - default: - return -1; - } // < switch(option_name) -} - -/// socket default handler getters - -UDP_FD::on_read_func UDP_FD::get_default_read_func() -{ - return [] (net::tcp::buffer_t) {}; -} -UDP_FD::on_write_func UDP_FD::get_default_write_func() -{ - return [] {}; -} -UDP_FD::on_except_func UDP_FD::get_default_except_func() -{ - return [] {}; -} diff --git a/src/posix/unistd.cpp b/src/posix/unistd.cpp deleted file mode 100644 index 8174a1a32b..0000000000 --- a/src/posix/unistd.cpp +++ /dev/null @@ -1,480 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static const int RNG_FD = 4; - -int open(const char* s, int oflag, ...) -{ - //const bool NON_BLOCKING = (oflag & 04000) != 0; - - if(strcmp(s, "/dev/random") == 0 || strcmp(s, "/dev/urandom") == 0) { - PRINT("open(%s, %d) = %d\n", s, oflag, RNG_FD); - return RNG_FD; - } - if (s == nullptr) { - PRINT("open(%s, %d) = %d\n", s, oflag, -1); - errno = EFAULT; - return -1; - } - if (s[0] == 0) { - PRINT("open(%s, %d) = %d\n", s, oflag, -1); - errno = ENOENT; - return -1; - } - try { - auto ent = fs::VFS::stat_sync(s); - if (ent.is_valid()) - { - auto& fd = FD_map::_open(ent); - PRINT("open(%s, %d) = %d\n", s, oflag, fd.get_id()); - return fd.get_id(); - } - errno = ENOENT; - } - catch (...) { - errno = ENOENT; - } - PRINT("open(%s, %d) = %d\n", s, oflag, -1); - return -1; -} - -int close(int fd) -{ - PRINT("close(%d)\n", fd); - if(fd == RNG_FD) { - return 0; - } - try { - return FD_map::_close(fd); - } - catch(const FD_not_found&) - { - errno = EBADF; - } - return -1; -} - -int read(int fd, void* buf, size_t len) -{ - PRINT("read(%d, %p, %lu)\n", fd, buf, len); - if (buf == nullptr) { - errno = EFAULT; - return -1; - } - if(fd == RNG_FD) { - rng_extract(buf, len); - return len; - } - try { - auto& fildes = FD_map::_get(fd); - return fildes.read(buf, len); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } - return 0; -} - -int write(int fd, const void* ptr, size_t len) -{ - if (fd < 4) { - OS::print((const char*) ptr, len); - return len; - } - PRINT("write(%d, %p, %lu)\n", fd, ptr, len); - if (fd == RNG_FD) { - rng_absorb(ptr, len); - return len; - } - try { - auto& fildes = FD_map::_get(fd); - return fildes.write(ptr, len); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -// read value of a symbolic link (which we don't have) -ssize_t readlink(const char* path, char*, size_t bufsiz) -{ - PRINT("readlink(%s, %lu) == -1\n", path, bufsiz); - errno = EBADF; - return -1; -} - -int fsync(int fildes) -{ - PRINT("fsync(%d)\n", fildes); - try { - (void) fildes; - //auto& fd = FD_map::_get(fildes); - // files should return 0, and others should not - return 0; - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -int fchown(int fd, uid_t, gid_t) -{ - PRINT("fchown(%d)\n", fd); - return -1; -} - -off_t lseek(int fd, off_t offset, int whence) -{ - PRINT("lseek(%d, %lu, %d)\n", fd, offset, whence); - try { - auto& fildes = FD_map::_get(fd); - return fildes.lseek(offset, whence); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -int isatty(int fd) -{ - if (fd == 1 || fd == 2 || fd == 3) { - PRINT("isatty(%d) = %d\n", fd, 1); - return 1; - } - PRINT("isatty(%d) = %d\n", fd, 0); - try { - auto& fildes = FD_map::_get(fd); - (void) fildes; - errno = ENOTTY; - return 0; - } - catch(const FD_not_found&) { - errno = EBADF; - return 0; - } -} - -#include -unsigned int sleep(unsigned int seconds) -{ - PRINT("sleep(%u) = %d\n", seconds, 0); - int64_t now = RTC::now(); - int64_t done = now + seconds; - while (now < done) { - OS::block(); - now = RTC::now(); - } - return 0; -} - -// todo: use fs::path as backing -static std::string cwd {"/"}; -const std::string& cwd_ref() { return cwd; } - -int chdir(const char *path) -// todo: handle relative path -// todo: handle .. -{ - PRINT("chdir(%s) = %d\n", path, -1); - if (not path or strlen(path) < 1) - { - errno = ENOENT; - return -1; - } - if (strcmp(path, ".") == 0) - { - return 0; - } - std::string desired_path; - if (*path != '/') - { - desired_path = cwd; - if (!(desired_path.back() == '/')) desired_path += "/"; - desired_path += path; - } - else - { - desired_path.assign(path); - } - try { - auto ent = fs::VFS::stat_sync(desired_path); - if (ent.is_dir()) - { - cwd = desired_path; - assert(cwd.front() == '/'); - assert(cwd.find("..") == std::string::npos); - return 0; - } - else - { - // path is not a dir - errno = ENOTDIR; - return -1; - } - } - catch (const fs::Err_not_found& e) { - errno = ENOTDIR; - return -1; - } -} - -char *getcwd(char *buf, size_t size) -{ - Expects(cwd.front() == '/'); - Expects(buf != nullptr); - - if (size == 0) - { - errno = EINVAL; - return nullptr; - } - if ((cwd.length() + 1) < size) - { - snprintf(buf, size, "%s", cwd.c_str()); - return buf; - } - else - { - errno = ERANGE; - return nullptr; - } -} - -int ftruncate(int fd, off_t length) -{ - PRINT("ftruncate(%d, %lu) = %d\n", fd, length, 1); - (void) fd; - (void) length; - // TODO: needs writable filesystem - errno = EBADF; - return -1; -} - -#include -#include -long sysconf(int name) -{ - PRINT("sysconf(%d)\n", name); - // for indeterminate limits, return -1, *don't* set errno - switch (name) { - case _SC_AIO_LISTIO_MAX: - case _SC_AIO_MAX: - case _SC_AIO_PRIO_DELTA_MAX: - return -1; - case _SC_ARG_MAX: - return ARG_MAX; - case _SC_ATEXIT_MAX: - return INT_MAX; - case _SC_CHILD_MAX: - return 0; - case _SC_CLK_TCK: - return 100; - case _SC_DELAYTIMER_MAX: - return -1; - case _SC_HOST_NAME_MAX: - return 255; - case _SC_LOGIN_NAME_MAX: - return 255; - case _SC_NGROUPS_MAX: - return 16; - case _SC_MQ_OPEN_MAX: - case _SC_MQ_PRIO_MAX: - return -1; - case _SC_OPEN_MAX: - return -1; - case _SC_PAGE_SIZE: // is also _SC_PAGESIZE - return OS::page_size(); - case _SC_RTSIG_MAX: - case _SC_SEM_NSEMS_MAX: - return -1; - case _SC_SEM_VALUE_MAX: - return INT_MAX; - case _SC_SIGQUEUE_MAX: - return -1; - case _SC_STREAM_MAX: // See APUE 2.5.1 - return FOPEN_MAX; - case _SC_TIMER_MAX: - return -1; - case _SC_TTY_NAME_MAX: - case _SC_TZNAME_MAX: - return 255; - case _SC_ADVISORY_INFO: - case _SC_BARRIERS: - case _SC_ASYNCHRONOUS_IO: - case _SC_CLOCK_SELECTION: - case _SC_CPUTIME: - return -1; - case _SC_MEMLOCK: - case _SC_MEMLOCK_RANGE: - case _SC_MESSAGE_PASSING: - return -1; - case _SC_MONOTONIC_CLOCK: - return 200809L; - case _SC_PRIORITIZED_IO: - case _SC_PRIORITY_SCHEDULING: - case _SC_RAW_SOCKETS: - return -1; - case _SC_REALTIME_SIGNALS: - return -1; - case _SC_SAVED_IDS: - return 1; - case _SC_SEMAPHORES: - case _SC_SHARED_MEMORY_OBJECTS: - return -1; - case _SC_SPAWN: - return 0; - case _SC_SPIN_LOCKS: - case _SC_SPORADIC_SERVER: - case _SC_SYNCHRONIZED_IO: - case _SC_THREAD_CPUTIME: - case _SC_THREAD_PRIO_INHERIT: - case _SC_THREAD_PRIO_PROTECT: - case _SC_THREAD_PRIORITY_SCHEDULING: - case _SC_THREAD_SPORADIC_SERVER: - case _SC_TIMEOUTS: - case _SC_TIMERS: - case _SC_TRACE: - case _SC_TRACE_EVENT_FILTER: - case _SC_TRACE_INHERIT: - case _SC_TRACE_LOG: - case _SC_TYPED_MEMORY_OBJECTS: - case _SC_2_FORT_DEV: - case _SC_2_PBS: - case _SC_2_PBS_ACCOUNTING: - case _SC_2_PBS_CHECKPOINT: - case _SC_2_PBS_LOCATE: - case _SC_2_PBS_MESSAGE: - case _SC_2_PBS_TRACK: - case _SC_XOPEN_REALTIME: - case _SC_XOPEN_REALTIME_THREADS: - case _SC_XOPEN_STREAMS: - return -1; - case _SC_XOPEN_UNIX: - return 1; - case _SC_XOPEN_VERSION: - return 600; - default: - errno = EINVAL; - return -1; - } -} - -uid_t getuid() { - PRINT("getuid() == 0\n"); - return 0; -} - -uid_t geteuid() { - PRINT("geteuid() == 0\n"); - return 0; -} - -gid_t getgid() { - PRINT("getgid() == 0\n"); - return 0; -} - -long fpathconf(int fd, int name) { - PRINT("fpathconf(%d, %d)\n", fd, name); - try { - auto& fildes = FD_map::_get(fd); - (void) fildes; - switch (name) { - case _PC_FILESIZEBITS: - return 64; - case _PC_LINK_MAX: - return -1; - case _PC_NAME_MAX: - return 255; - case _PC_PATH_MAX: - return 1024; - case _PC_SYMLINK_MAX: - return -1; - case _PC_CHOWN_RESTRICTED: - return -1; - case _PC_NO_TRUNC: - return -1; - case _PC_VDISABLE: - return -1; - case _PC_ASYNC_IO: - return 0; - case _PC_SYNC_IO: - return 0; - default: - errno = EINVAL; - return -1; - } - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } -} - -long pathconf(const char *path, int name) { - PRINT("pathconf(%s, %d)\n", path, name); - int fd = open(path, O_RDONLY); - if (fd == -1) { - errno = ENOENT; - return -1; - } - long res = fpathconf(fd, name); - close(fd); - return res; -} - -int execve(const char*, - char* const*, - char* const*) -{ - panic("SYSCALL EXECVE NOT SUPPORTED"); - return -1; -} - -int fork() { - panic("SYSCALL FORK NOT SUPPORTED"); - return -1; -} - -int getpid() { - debug("SYSCALL GETPID Dummy, returning 1"); - return 1; -} - -int link(const char*, const char*) { - panic("SYSCALL LINK unsupported"); - return -1; -} - -int unlink(const char*) { - panic("SYSCALL UNLINK unsupported"); - return -1; -} diff --git a/src/posix/utime.cpp b/src/posix/utime.cpp deleted file mode 100644 index e73b5796ba..0000000000 --- a/src/posix/utime.cpp +++ /dev/null @@ -1,18 +0,0 @@ -#include -#include -#include -#include - -int utime(const char *filename, const struct utimbuf *times) -{ - printf("utime(%s, %p) called, not supported\n", filename, times); - errno = ENOTSUP; - return -1; -} - -int utimes(const char *filename, const struct timeval times[2]) -{ - printf("utimes(%s, %p) called, not supported\n", filename, times); - errno = ENOTSUP; - return -1; -} From 1f2d14f3d861b405ab943ae378bfdc524a8f2193 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 9 Jan 2018 19:13:38 +0100 Subject: [PATCH 017/723] musl: Linux syscall API replacement + update service build system --- api/posix/stddef.h | 3 + api/syscalls.h | 278 +++++++++++++++++++++++++ api/util/tar.hpp | 38 ++++ cmake/cross_compiled_libraries.txt | 23 +- cmake/post.service.cmake | 64 ++++-- etc/build_llvm.sh | 5 +- etc/build_musl.sh | 14 +- etc/create_binary_bundle.sh | 2 +- etc/musl/syscall.h | 223 ++++++++++++++++++++ install.sh | 9 +- lib/LiveUpdate/update.cpp | 2 +- mod/CMakeLists.txt | 2 +- src/crt/c_abi.c | 56 ++--- src/kernel/heap.cpp | 20 +- src/kernel/syscalls.cpp | 2 + src/platform/x86_nano/kernel_start.cpp | 15 +- src/platform/x86_nano/platform.cpp | 3 + src/platform/x86_pc/os.cpp | 4 +- src/platform/x86_pc/serial1.cpp | 5 + vmbuild/CMakeLists.txt | 1 + vmbuild/vmbuild.cpp | 2 +- 21 files changed, 688 insertions(+), 83 deletions(-) create mode 100644 api/syscalls.h create mode 100644 etc/musl/syscall.h diff --git a/api/posix/stddef.h b/api/posix/stddef.h index 3044a26246..50031edd2e 100644 --- a/api/posix/stddef.h +++ b/api/posix/stddef.h @@ -1,2 +1,5 @@ +#define _DYNAMIC + + #define _GCC_MAX_ALIGN_T #include_next diff --git a/api/syscalls.h b/api/syscalls.h new file mode 100644 index 0000000000..d199f40e73 --- /dev/null +++ b/api/syscalls.h @@ -0,0 +1,278 @@ +#pragma once + +#define __includeos(num, ...) extern long syscall_##num(long,...) + +extern long syscall_n(long,...); +extern long syscall_SYS_brk(void*); +extern long syscall_SYS_close(long,...); +extern long syscall_SYS_epoll_wait(long,...); +extern long syscall_SYS_exit_group(long,...); +extern long syscall_SYS_exit(long,...); +extern long syscall_SYS_fadvise(long,...); +extern long syscall_SYS_fallocate(long,...); +extern long syscall_SYS_fcntl(long,...); +extern long syscall_SYS_flistxattr(long,...); +extern long syscall_SYS_fork(); +extern long syscall_SYS_fremovexattr(long,...); +extern long syscall_SYS_fsetxattr(long,...); +extern long syscall_SYS_futex(long, ...); +extern long syscall_SYS_getdents(long,...); +extern long syscall_SYS_getegid(); +extern long syscall_SYS_geteuid(); +extern long syscall_SYS_getgid(); +extern long syscall_SYS_getpid(); +extern long syscall_SYS_getppid(); +extern long syscall_SYS_gettid(); +extern long syscall_SYS_getuid(); +extern long syscall_SYS_inotify_init(); +extern long syscall_SYS_ioctl(long, long, long, ...); +extern long syscall_SYS_lremovexattr(long,...); +extern long syscall_SYS_mmap2(long,...); +extern long syscall_SYS_msgctl(long,...); +extern long syscall_SYS_msgget(long,...); +extern long syscall_SYS_munlockall(); +extern long syscall_SYS_pause(); +extern long syscall_SYS_poll(long,...); +extern long syscall_SYS_removexattr(long,...); +extern long syscall_SYS_rt_sigqueueinfo(long, ...); +extern long syscall_SYS_sched_getaffinity(long, ...); +extern long syscall_SYS_sched_yield(); +extern long syscall_SYS_semctl(long,...); +extern long syscall_SYS_semget(long,...); +extern long syscall_SYS_semop(long,...); +extern long syscall_SYS_semtimedop(long,...); +extern long syscall_SYS_setsid(); +extern long syscall_SYS_set_tid_address(long,...); +extern long syscall_SYS_shmat(long,...); +extern long syscall_SYS_sync(); +extern long syscall_SYS_vhangup(); +extern int syscall_SYS_open(const char *path, int oflag, ... ); + +__includeos(SYS_access); +__includeos(SYS_acct); +__includeos(SYS_adjtimex); +__includeos(SYS_arch_prctl); +__includeos(SYS_capget); +__includeos(SYS_capset); +__includeos(SYS_chdir); +__includeos(SYS_chmod); +__includeos(SYS_chown); +__includeos(SYS_chroot); +__includeos(SYS_clock_adjtime); +__includeos(SYS_clock_getres); +__includeos(SYS_clock_gettime); +__includeos(SYS_clock_nanosleep); +__includeos(SYS_clock_settime); +__includeos(SYS_delete_module); +__includeos(SYS_dup); +__includeos(SYS_dup2); +__includeos(SYS_dup3); +__includeos(SYS_epoll_create); +__includeos(SYS_epoll_create1); +__includeos(SYS_epoll_ctl); +__includeos(SYS_epoll_pwait); +__includeos(SYS_eventfd); +__includeos(SYS_eventfd2); +__includeos(SYS_execve); +__includeos(SYS_faccessat); +__includeos(SYS_fanotify_init); +__includeos(SYS_fanotify_mark); +__includeos(SYS_fchdir); +__includeos(SYS_fchmod); +__includeos(SYS_fchmodat); +__includeos(SYS_fchown); +__includeos(SYS_fchownat); +__includeos(SYS_fdatasync); +__includeos(SYS_fgetxattr); +__includeos(SYS_flock); +__includeos(SYS_fstat); +__includeos(SYS_fstatat); +__includeos(SYS_fstatfs); +__includeos(SYS_fstatfs64); +__includeos(SYS_fsync); +__includeos(SYS_ftruncate); +__includeos(SYS_futimesat); +__includeos(SYS_getcpu); +__includeos(SYS_getcwd); +__includeos(SYS_getgroups); +__includeos(SYS_getitimer); +__includeos(SYS_getpgid); +__includeos(SYS_getpriority); +__includeos(SYS_getresgid); +__includeos(SYS_getresuid); +__includeos(SYS_getrlimit); +__includeos(SYS_getrusage); +__includeos(SYS_getsid); +__includeos(SYS_gettimeofday); +__includeos(SYS_getxattr); +__includeos(SYS_init_module); +__includeos(SYS_inotify_add_watch); +__includeos(SYS_inotify_init1); +__includeos(SYS_inotify_rm_watch); +__includeos(SYS_ioperm); +__includeos(SYS_iopl); +__includeos(SYS_ipc); +__includeos(SYS_kill); +__includeos(SYS_lchown); +__includeos(SYS_lgetxattr); +__includeos(SYS_link); +__includeos(SYS_link175); +__includeos(SYS_linkat); +__includeos(SYS_listxattr); +__includeos(SYS_llistxattr); +__includeos(SYS__llseek); +__includeos(SYS_lseek); +__includeos(SYS_lsetxattr); +__includeos(SYS_lstat); +__includeos(SYS_madvise); +__includeos(SYS_mincore); +__includeos(SYS_mkdir); +__includeos(SYS_mkdirat); +__includeos(SYS_mknod); +__includeos(SYS_mknodat); +__includeos(SYS_mlock); +__includeos(SYS_mlockall); +__includeos(SYS_mmap); +__includeos(SYS_mount); +__includeos(SYS_mprotect); +__includeos(SYS_mq_getsetattr); +__includeos(SYS_mq_notify); +__includeos(SYS_mq_open); +__includeos(SYS_mq_timedreceive); +__includeos(SYS_mq_timedsend); +__includeos(SYS_mq_unlink); +__includeos(SYS_mremap); +__includeos(SYS_msgrcv); +__includeos(SYS_msgsnd); +__includeos(SYS_msync); +__includeos(SYS_munlock); +__includeos(SYS_munmap); +__includeos(SYS_nanosleep); +__includeos(SYS_nice); +__includeos(SYS_openat); +__includeos(SYS_personality); +__includeos(SYS_pipe); +__includeos(SYS_pipe2); +__includeos(SYS_pivot_root); +__includeos(SYS_ppoll); +__includeos(SYS_prctl); +__includeos(SYS_pread); +__includeos(SYS_preadv); +__includeos(SYS_prlimit64); +__includeos(SYS_process_vm_readv); +__includeos(SYS_process_vm_writev); +__includeos(SYS_pselect6); +__includeos(SYS_ptrace); +__includeos(SYS_pwrite); +__includeos(SYS_pwritev); +__includeos(SYS_quotactl); +__includeos(SYS_read); +__includeos(SYS_readahead); +__includeos(SYS_readlink); +__includeos(SYS_readlinkat); +__includeos(SYS_readv); +__includeos(SYS_reboot); +__includeos(SYS_remap_file_pages); +__includeos(SYS_rename); +__includeos(SYS_renameat); +__includeos(SYS_rmdir); +__includeos(SYS_rt_sigaction); +__includeos(SYS_rt_sigpending); +__includeos(SYS_rt_sigprocmask); +__includeos(SYS_rt_sigsuspend); +__includeos(SYS_rt_sigtimedwait); +__includeos(SYS_sched_getparam); +__includeos(SYS_sched_get_priority_max); +__includeos(SYS_sched_get_priority_min); +__includeos(SYS_sched_getscheduler); +__includeos(SYS_sched_rr_get_interval); +__includeos(SYS_sched_setaffinity); +__includeos(SYS_sched_setparam); +__includeos(SYS_sched_setscheduler); +__includeos(SYS_select); +__includeos(SYS_sendfile); +__includeos(SYS_setdomainname); +__includeos(SYS_setfsgid); +__includeos(SYS_setfsuid); +__includeos(SYS_setgid); +__includeos(SYS_setgroups); +__includeos(SYS_sethostname); +__includeos(SYS_setitimer); +__includeos(SYS_setns); +__includeos(SYS_setpgid); +__includeos(SYS_setpriority); +__includeos(SYS_setregid); +__includeos(SYS_setreuid); +__includeos(SYS_setrlimit); +__includeos(SYS_set_robust_list); +__includeos(SYS_settimeofday); +__includeos(SYS_setuid); +__includeos(SYS_setxattr); +__includeos(SYS_shmctl); +__includeos(SYS_shmdt); +__includeos(SYS_shmget); +__includeos(SYS_sigaltstack); +__includeos(SYS_signalfd); +__includeos(SYS_signalfd4); +__includeos(SYS_socketcall); +__includeos(SYS_splice); +__includeos(SYS_stat); +__includeos(SYS_statfs); +__includeos(SYS_statfs64); +__includeos(SYS_swapoff); +__includeos(SYS_swapon); +__includeos(SYS_symlink); +__includeos(SYS_symlinkat); +__includeos(SYS_sync_file_range); +__includeos(SYS_syncfs); +__includeos(SYS_sysinfo); +__includeos(SYS_syslog); +__includeos(SYS_tee); +__includeos(SYS_tgkill); +__includeos(SYS_timer_create); +__includeos(SYS_timer_delete); +__includeos(SYS_timerfd_create); +__includeos(SYS_timerfd_gettime); +__includeos(SYS_timerfd_settime); +__includeos(SYS_timer_getoverrun); +__includeos(SYS_timer_gettime); +__includeos(SYS_timer_settime); +__includeos(SYS_times); +__includeos(SYS_tkill); +__includeos(SYS_truncate); +__includeos(SYS_umask); +__includeos(SYS_umount2); +__includeos(SYS_uname); +__includeos(SYS_unlink); +__includeos(SYS_unlinkat); +__includeos(SYS_unshare); +__includeos(SYS_utimensat); +__includeos(SYS_utimes); +__includeos(SYS_vmsplice); +__includeos(SYS_wait4); +__includeos(SYS_waitid); +__includeos(SYS_write); +__includeos(SYS_writev); + +int socketcall_socket(int,...); +int socketcall_bind(int,...); +int socketcall_connect(int,...); +int socketcall_listen(int,...); +int socketcall_accept(int,...); +int socketcall_getsockname(int,...); +int socketcall_getpeername(int,...); +int socketcall_socketpair(int,...); +int socketcall_send(int,...); +int socketcall_recv(int,...); +int socketcall_sendto(int,...); +int socketcall_recvfrom(int,...); +int socketcall_shutdown(int,...); +int socketcall_setsockopt(int,...); +int socketcall_getsockopt(int,...); +int socketcall_sendmsg(int,...); +int socketcall_recvmsg(int,...); +int socketcall_accept4(int,...); +int socketcall_recvmmsg(int,...); +int syscall_SYS_recvmmsg(int,...); +int syscall_SYS_sendmmsg(int,...); +//int socketcall_sendmmsg(int,...); diff --git a/api/util/tar.hpp b/api/util/tar.hpp index 97d3f5cfbd..bddb20b2ba 100644 --- a/api/util/tar.hpp +++ b/api/util/tar.hpp @@ -33,6 +33,44 @@ extern uint8_t _binary_input_bin_start; extern uintptr_t _binary_input_bin_size; +#define LENGTH_NAME 100 +#define LENGTH_MODE 8 +#define LENGTH_UID 8 +#define LENGTH_GID 8 +#define LENGTH_SIZE 12 +#define LENGTH_MTIME 12 +#define LENGTH_CHECKSUM 8 +#define LENGTH_TYPEFLAG 1 +#define LENGTH_LINKNAME 100 +#define LENGTH_MAGIC 6 +#define LENGTH_VERSION 2 +#define LENGTH_UNAME 32 +#define LENGTH_GNAME 32 +#define LENGTH_DEVMAJOR 8 +#define LENGTH_DEVMINOR 8 +#define LENGTH_PREFIX 155 +#define LENGTH_PAD 12 + +struct Tar_header { + char name[LENGTH_NAME]; // Name of header file entry + char mode[LENGTH_MODE]; // Permission and mode bits + char uid[LENGTH_UID]; // User ID of owner + char gid[LENGTH_GID]; // Group ID of owner + char size[LENGTH_SIZE]; // File size in bytes (octal base) + char mod_time[LENGTH_MTIME]; // Last modification time in numeric Unix time format (octal) + char checksum[LENGTH_CHECKSUM]; // Checksum for header record (6 digit octal number with leading zeroes) + char typeflag; // Type of header entry + char linkname[LENGTH_LINKNAME]; // Target name of link + char magic[LENGTH_MAGIC]; // Ustar indicator + char version[LENGTH_VERSION]; // Ustar version + char uname[LENGTH_UNAME]; // User name of owner + char gname[LENGTH_GNAME]; // Group name of owner + char devmajor[LENGTH_DEVMAJOR]; // Major number of character or block device + char devminor[LENGTH_DEVMINOR]; // Minor number of character or block device + char prefix[LENGTH_PREFIX]; // Prefix for file name + char pad[LENGTH_PAD]; +}; + namespace tar { static const int SECTOR_SIZE = 512; diff --git a/cmake/cross_compiled_libraries.txt b/cmake/cross_compiled_libraries.txt index 3c705cceba..15d34c855c 100644 --- a/cmake/cross_compiled_libraries.txt +++ b/cmake/cross_compiled_libraries.txt @@ -74,16 +74,18 @@ set(LIBCXX_LIB_DIR ${PRECOMPILED_DIR}/libcxx/) add_library(libcxx STATIC IMPORTED) add_library(libcxxabi STATIC IMPORTED) +add_library(libunwind STATIC IMPORTED) add_dependencies(libcxx PrecompiledLibraries) add_dependencies(libcxxabi PrecompiledLibraries) set_target_properties(libcxx PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libc++.a) set_target_properties(libcxxabi PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libc++abi.a) +set_target_properties(libunwind PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libunwind.a) set(MUSL_INCLUDE_DIR ${PRECOMPILED_DIR}/musl/include/) -set(MUSL_LIB_DIR ${PRECOMPILED_DIR}/musl/) +set(MUSL_LIB_DIR ${PRECOMPILED_DIR}/musl/lib) -set(LIBGCC_LIB_DIR ${PRECOMPILED_DIR}/libgcc/) +#set(LIBGCC_LIB_DIR ${PRECOMPILED_DIR}/libgcc/) add_library(libc STATIC IMPORTED) set_target_properties(libc PROPERTIES IMPORTED_LOCATION ${MUSL_LIB_DIR}/libc.a) @@ -93,14 +95,23 @@ add_library(libm STATIC IMPORTED) set_target_properties(libm PROPERTIES IMPORTED_LOCATION ${MUSL_LIB_DIR}/libm.a) add_dependencies(libm PrecompiledLibraries) -set(CRTEND ${PRECOMPILED_DIR}/crt/crtend.o) -set(CRTBEGIN ${PRECOMPILED_DIR}/crt/crtbegin.o) +#set(CRTEND ${PRECOMPILED_DIR}/crt/crtend.o) +#set(CRTBEGIN ${PRECOMPILED_DIR}/crt/crtbegin.o) # installation instructions install(DIRECTORY ${LIBCXX_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/libcxx) install(DIRECTORY ${MUSL_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/musl) -install(FILES ${CRTEND} ${CRTBEGIN} DESTINATION includeos/${ARCH}/lib) +#install(FILES ${CRTEND} ${CRTBEGIN} DESTINATION includeos/${ARCH}/lib) -install(FILES ${MUSL_LIB_DIR}/libc.a ${LIBCXX_LIB_DIR}/libc++.a ${LIBCXX_LIB_DIR}/libc++abi.a DESTINATION includeos/${ARCH}/lib) +file(GLOB musl_libs ${MUSL_LIB_DIR}/*.a) +file(GLOB musl_objs ${MUSL_LIB_DIR}/*.o) + +install(FILES + ${musl_libs} + ${musl_objs} + ${LIBCXX_LIB_DIR}/libc++.a + ${LIBCXX_LIB_DIR}/libc++abi.a + ${LIBCXX_LIB_DIR}/libunwind.a + DESTINATION includeos/${ARCH}/lib) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 5dfc1f3d4c..87f10c2a69 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -32,10 +32,19 @@ else() endif() enable_language(ASM_NASM) + +# Find compiler-rt (LLVM's libgcc alternative) +execute_process(COMMAND ${CMAKE_CXX_COMPILER} + -nostdlib -rtlib=compiler-rt -print-libgcc-file-name -target ${TRIPLE} + OUTPUT_VARIABLE compiler_rt_path OUTPUT_STRIP_TRAILING_WHITESPACE) + +message(STATUS "Using compiler-rt at ${compiler_rt_path}") +#set(compiler_rt_path "/usr/lib/llvm-4.0/bin/../lib/clang/4.0.0/lib/linux/libclang_rt.builtins-i686.a") + # Various global defines # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures # * _GNU_SOURCE enables POSIX-extensions in newlib, such as strnlen. ("everything newlib has", ref. cdefs.h) -set(CAPABS "${CAPABS} -fstack-protector-strong -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_GNU_SOURCE -DSERVICE=\"\\\"${BINARY}\\\"\" -DSERVICE_NAME=\"\\\"${SERVICE_NAME}\\\"\"") +set(CAPABS "${CAPABS} --rtlib=compiler-rt -fstack-protector-strong -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE -DSERVICE=\"\\\"${BINARY}\\\"\" -DSERVICE_NAME=\"\\\"${SERVICE_NAME}\\\"\"") set(WARNS "-Wall -Wextra") #-pedantic # Compiler optimization @@ -214,7 +223,7 @@ endforeach() include_directories(${LOCAL_INCLUDES}) include_directories(${INSTALL_LOC}/api/posix) include_directories(${INSTALL_LOC}/${ARCH}/include/libcxx) -include_directories(${INSTALL_LOC}/${ARCH}/include/newlib) +include_directories(${INSTALL_LOC}/${ARCH}/include/musl) if ("${PLATFORM}" STREQUAL "x86_solo5") include_directories(${INSTALL_LOC}/${ARCH}/include/solo5) @@ -252,7 +261,7 @@ if ("${PLATFORM}" STREQUAL "x86_solo5") set(PRE_BSS_SIZE "--defsym PRE_BSS_AREA=0x200000") endif() -set(LDFLAGS "-nostdlib -melf_${ELF} -N --eh-frame-hdr ${STRIP_LV} --script=${INSTALL_LOC}/${ARCH}/linker.ld ${PRE_BSS_SIZE} ${INSTALL_LOC}/${ARCH}/lib/crtbegin.o") +set(LDFLAGS "-nostdlib -melf_${ELF} -N --eh-frame-hdr ${STRIP_LV} --script=${INSTALL_LOC}/${ARCH}/linker.ld ${PRE_BSS_SIZE} ") #${INSTALL_LOC}/${ARCH}/lib/crtbegin.o") set_target_properties(service PROPERTIES LINK_FLAGS "${LDFLAGS}") @@ -296,23 +305,27 @@ set_target_properties(libosdeps PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${AR add_library(libcxx STATIC IMPORTED) add_library(cxxabi STATIC IMPORTED) +add_library(libunwind STATIC IMPORTED) +add_library(compiler_rt STATIC IMPORTED) set_target_properties(libcxx PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(libcxx PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libc++.a) set_target_properties(cxxabi PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(cxxabi PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libc++abi.a) +set_target_properties(libunwind PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(libunwind PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libunwind.a) +set_target_properties(compiler_rt PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(compiler_rt PROPERTIES IMPORTED_LOCATION ${compiler_rt_path}) + + add_library(libc STATIC IMPORTED) set_target_properties(libc PROPERTIES LINKER_LANGUAGE C) set_target_properties(libc PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libc.a) -add_library(libm STATIC IMPORTED) -set_target_properties(libm PROPERTIES LINKER_LANGUAGE C) -set_target_properties(libm PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libm.a) -add_library(libg STATIC IMPORTED) -set_target_properties(libg PROPERTIES LINKER_LANGUAGE C) -set_target_properties(libg PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libg.a) -add_library(libgcc STATIC IMPORTED) -set_target_properties(libgcc PROPERTIES LINKER_LANGUAGE C) -set_target_properties(libgcc PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libgcc.a) + +add_library(libpthread STATIC IMPORTED) +set_target_properties(libpthread PROPERTIES LINKER_LANGUAGE C) +set_target_properties(libpthread PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libpthread.a) + if ("${PLATFORM}" STREQUAL "x86_solo5") add_library(solo5 STATIC IMPORTED) @@ -419,23 +432,32 @@ target_link_libraries(service libplatform libarch libos + libc + cxxabi + libpthread + libunwind + libcxx libplatform libarch libos - libplatform - - cxxabi libc - libos + cxxabi + libpthread + libunwind libcxx - libm - libg - libgcc + libplatform + libarch + libos + libc + cxxabi + libpthread + libunwind + libcxx + compiler_rt - ${INSTALL_LOC}/${ARCH}/lib/crtend.o - --whole-archive crtn --no-whole-archive + #--whole-archive crtn --no-whole-archive ) # write binary location to known file file(WRITE ${CMAKE_BINARY_DIR}/binary.txt ${BINARY}) diff --git a/etc/build_llvm.sh b/etc/build_llvm.sh index d63df2d61f..c1596dc552 100755 --- a/etc/build_llvm.sh +++ b/etc/build_llvm.sh @@ -91,13 +91,16 @@ cmake -GNinja $OPTS \ -DLIBCXX_TARGET_TRIPLE=$TRIPLE \ -DLIBCXX_BUILD_32_BITS=OFF \ -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ + -DLIBCXX_USE_COMPILER_RT=ON \ -DLIBCXX_CXX_ABI=libcxxabi \ -DLIBCXXABI_TARGET_TRIPLE=$TRIPLE \ -DLIBCXXABI_ENABLE_THREADS=ON \ -DLIBCXXABI_HAS_PTHREAD_API=ON \ -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ -DLIBUNWIND_TARGET_TRIPLE=$TRIPLE \ - -DLIBUNWIND_ENABLE_SHARED=OFF + -DLIBUNWIND_TARGET_TRIPLE=$TRIPLE \ + -DLIBUNWIND_ENABLE_SHARED=OFF \ + -DCOMPILER_RT_BUILD_SANITIZERS=OFF # MAKE diff --git a/etc/build_musl.sh b/etc/build_musl.sh index 3732aa3945..6d5e4dd4d6 100755 --- a/etc/build_musl.sh +++ b/etc/build_musl.sh @@ -10,12 +10,22 @@ if [ ! -d musl-$musl_version ]; then git clone git://git.musl-libc.org/musl/ || true fi +pushd musl + +# Replace syscall API +cp $INCLUDEOS_SRC/api/syscalls.h src/internal/includeos_syscalls.h +cp $INCLUDEOS_SRC/etc/musl/syscall.h src/internal/ + +rm -f arch/x86_64/syscall_arch.h +rm -f arch/i386/syscall_arch.h + +# ln -s $INCLUDEOS_SRC/api/arch/syscalls.h arch/x86_64/syscall_arch.h +# ln -s $INCLUDEOS_SRC/api/arch/syscalls.h arch/i386/syscall_arch.h -pushd musl git checkout $musl_version make distclean || true -# CC="$CC -m32" + export CFLAGS="$CFLAGS -target $ARCH-pc-linux-elf" ./configure --prefix=$TEMP_INSTALL_DIR/$TARGET --disable-shared --enable-optimize=* --target=$TARGET make $num_jobs diff --git a/etc/create_binary_bundle.sh b/etc/create_binary_bundle.sh index 4017a49f21..035900aefc 100755 --- a/etc/create_binary_bundle.sh +++ b/etc/create_binary_bundle.sh @@ -97,7 +97,7 @@ popd # Where to place the installation bundle DIR_NAME="IncludeOS_dependencies" OUTFILE="${DIR_NAME}_$filename_tag.tar.gz" -BUNDLE_PATH=${BUNDLE_PATH:-~} +BUNDLE_PATH=${BUNDLE_PATH:-$BUILD_DIR} function do_build { echo -e "\n\n >>> Building bundle for ${ARCH} \n" diff --git a/etc/musl/syscall.h b/etc/musl/syscall.h new file mode 100644 index 0000000000..420f4138ea --- /dev/null +++ b/etc/musl/syscall.h @@ -0,0 +1,223 @@ +#ifndef _INTERNAL_SYSCALL_H +#define _INTERNAL_SYSCALL_H + +#include +//#include "syscall_arch.h"y +#include "includeos_syscalls.h" + + +#define __SYSCALL_LL_E(x) \ +((union { long long ll; long l[2]; }){ .ll = x }).l[0], \ +((union { long long ll; long l[2]; }){ .ll = x }).l[1] +#define __SYSCALL_LL_O(x) __SYSCALL_LL_E((x)) + + +#ifndef SYSCALL_RLIM_INFINITY +#define SYSCALL_RLIM_INFINITY (~0ULL) +#endif + +#ifndef SYSCALL_MMAP2_UNIT +#define SYSCALL_MMAP2_UNIT 4096ULL +#endif + +#ifndef __SYSCALL_LL_PRW +#define __SYSCALL_LL_PRW(x) __SYSCALL_LL_O(x) +#endif + +#ifndef __scc +#define __scc(X) ((long) (X)) +typedef long syscall_arg_t; +#endif + +__attribute__((visibility("hidden"))) +long __syscall_ret(unsigned long), __syscall(syscall_arg_t, ...), + __syscall_cp(syscall_arg_t, syscall_arg_t, syscall_arg_t, syscall_arg_t, + syscall_arg_t, syscall_arg_t, syscall_arg_t); + +#define __SYSCALL_NARGS_X(a,b,c,d,e,f,g,h,n,...) n +#define __SYSCALL_NARGS(...) __SYSCALL_NARGS_X(__VA_ARGS__,7,6,5,4,3,2,1,0,) +#define __SYSCALL_CONCAT_X(a,b) a##b +#define __SYSCALL_CONCAT(a,b) __SYSCALL_CONCAT_X(a,b) +#define __syscall(a,...) syscall_##a(__VA_ARGS__) +#define syscall(a,...) __syscall_ret(syscall_##a(__VA_ARGS__)) + + +#define socketcall __socketcall +#define socketcall_cp __socketcall_cp + +#define __syscall_cp syscall +#define syscall_cp syscall + +#define __socketcall(nm,a,b,c,d,e,f) socketcall_##nm \ + ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f }) +#define __socketcall_cp(nm,a,b,c,d,e,f) __syscall_ret(__socketcall(nm,a,b,c,d,e,f)) + +/* fixup legacy 16-bit junk */ + +#ifdef SYS_getuid32 +#undef SYS_lchown +#undef SYS_getuid +#undef SYS_getgid +#undef SYS_geteuid +#undef SYS_getegid +#undef SYS_setreuid +#undef SYS_setregid +#undef SYS_getgroups +#undef SYS_setgroups +#undef SYS_fchown +#undef SYS_setresuid +#undef SYS_getresuid +#undef SYS_setresgid +#undef SYS_getresgid +#undef SYS_chown +#undef SYS_setuid +#undef SYS_setgid +#undef SYS_setfsuid +#undef SYS_setfsgid +#define SYS_lchown SYS_lchown32 +#define SYS_getuid SYS_getuid32 +#define SYS_getgid SYS_getgid32 +#define SYS_geteuid SYS_geteuid32 +#define SYS_getegid SYS_getegid32 +#define SYS_setreuid SYS_setreuid32 +#define SYS_setregid SYS_setregid32 +#define SYS_getgroups SYS_getgroups32 +#define SYS_setgroups SYS_setgroups32 +#define SYS_fchown SYS_fchown32 +#define SYS_setresuid SYS_setresuid32 +#define SYS_getresuid SYS_getresuid32 +#define SYS_setresgid SYS_setresgid32 +#define SYS_getresgid SYS_getresgid32 +#define SYS_chown SYS_chown32 +#define SYS_setuid SYS_setuid32 +#define SYS_setgid SYS_setgid32 +#define SYS_setfsuid SYS_setfsuid32 +#define SYS_setfsgid SYS_setfsgid32 +#endif + + +/* fixup legacy 32-bit-vs-lfs64 junk */ + +#ifdef SYS_fcntl64 +#undef SYS_fcntl +#define SYS_fcntl SYS_fcntl64 +#endif + +#ifdef SYS_getdents64 +#undef SYS_getdents +#define SYS_getdents SYS_getdents64 +#endif + +#ifdef SYS_ftruncate64 +#undef SYS_ftruncate +#undef SYS_truncate +#define SYS_ftruncate SYS_ftruncate64 +#define SYS_truncate SYS_truncate64 +#endif + +#ifdef SYS_stat64 +#undef SYS_stat +#define SYS_stat SYS_stat64 +#endif + +#ifdef SYS_fstat64 +#undef SYS_fstat +#define SYS_fstat SYS_fstat64 +#endif + +#ifdef SYS_lstat64 +#undef SYS_lstat +#define SYS_lstat SYS_lstat64 +#endif + +#ifdef SYS_statfs64 +#undef SYS_statfs +#define SYS_statfs SYS_statfs64 +#endif + +#ifdef SYS_fstatfs64 +#undef SYS_fstatfs +#define SYS_fstatfs SYS_fstatfs64 +#endif + +#if defined(SYS_newfstatat) +#undef SYS_fstatat +#define SYS_fstatat SYS_newfstatat +#elif defined(SYS_fstatat64) +#undef SYS_fstatat +#define SYS_fstatat SYS_fstatat64 +#endif + +#ifdef SYS_ugetrlimit +#undef SYS_getrlimit +#define SYS_getrlimit SYS_ugetrlimit +#endif + +#ifdef SYS__newselect +#undef SYS_select +#define SYS_select SYS__newselect +#endif + +#ifdef SYS_pread64 +#undef SYS_pread +#undef SYS_pwrite +#define SYS_pread SYS_pread64 +#define SYS_pwrite SYS_pwrite64 +#endif + +#ifdef SYS_fadvise64_64 +#undef SYS_fadvise +#define SYS_fadvise SYS_fadvise64_64 +#elif defined(SYS_fadvise64) +#undef SYS_fadvise +#define SYS_fadvise SYS_fadvise64 +#endif + +#ifdef SYS_sendfile64 +#undef SYS_sendfile +#define SYS_sendfile SYS_sendfile64 +#endif + +/* socketcall calls */ + + +#define __SC_socket 1 +#define __SC_bind 2 +#define __SC_connect 3 +#define __SC_listen 4 +#define __SC_accept 5 +#define __SC_getsockname 6 +#define __SC_getpeername 7 +#define __SC_socketpair 8 +#define __SC_send 9 +#define __SC_recv 10 +#define __SC_sendto 11 +#define __SC_recvfrom 12 +#define __SC_shutdown 13 +#define __SC_setsockopt 14 +#define __SC_getsockopt 15 +#define __SC_sendmsg 16 +#define __SC_recvmsg 17 +#define __SC_accept4 18 +#define __SC_recvmmsg 19 +#define __SC_sendmmsg 20 + +#ifdef SYS_open +#define __sys_open2(x,pn,fl) __syscall2(SYS_open, pn, (fl)|O_LARGEFILE) +#define __sys_open3(x,pn,fl,mo) __syscall3(SYS_open, pn, (fl)|O_LARGEFILE, mo) +#define __sys_open_cp2(x,pn,fl) __syscall_cp2(SYS_open, pn, (fl)|O_LARGEFILE) +#define __sys_open_cp3(x,pn,fl,mo) __syscall_cp3(SYS_open, pn, (fl)|O_LARGEFILE, mo) +#else +#define __sys_open2(x,pn,fl) __syscall3(SYS_openat, AT_FDCWD, pn, (fl)|O_LARGEFILE) +#define __sys_open3(x,pn,fl,mo) __syscall4(SYS_openat, AT_FDCWD, pn, (fl)|O_LARGEFILE, mo) +#define __sys_open_cp2(x,pn,fl) __syscall_cp3(SYS_openat, AT_FDCWD, pn, (fl)|O_LARGEFILE) +#define __sys_open_cp3(x,pn,fl,mo) __syscall_cp4(SYS_openat, AT_FDCWD, pn, (fl)|O_LARGEFILE, mo) +#endif + +#define __sys_open syscall_SYS_open +#define sys_open(...) __syscall_ret(__sys_open(__VA_ARGS__)) + +#define __sys_open_cp __sys_open +#define sys_open_cp(...) __syscall_ret(__sys_open_cp(__VA_ARGS__)) + +#endif diff --git a/install.sh b/install.sh index d8369991d0..285bf69aa9 100755 --- a/install.sh +++ b/install.sh @@ -124,9 +124,12 @@ else else dependency_level=build fi - echo ">>> Dependencies required:" - if ! ./etc/install_dependencies_linux.sh -s $SYSTEM -r $RELEASE -c -d $dependency_level; then - missing_dependencies=1 + if [ $do_packages ] + then + echo ">>> Dependencies required:" + if ! ./etc/install_dependencies_linux.sh -s $SYSTEM -r $RELEASE -c -d $dependency_level; then + missing_dependencies=1 + fi fi fi diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index 12e2574693..c3120e1a11 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #include "storage.hpp" #include diff --git a/mod/CMakeLists.txt b/mod/CMakeLists.txt index d2c9278a5b..f11c921aa9 100644 --- a/mod/CMakeLists.txt +++ b/mod/CMakeLists.txt @@ -1,6 +1,6 @@ include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) -include_directories(${NEWLIB_INCLUDE_DIR}) +include_directories(${MUSL_INCLUDE_DIR}) include_directories(${SOLO5_INCLUDE_DIR}) include_directories(${INCLUDEOS_ROOT}/src/include) include_directories(${INCLUDEOS_ROOT}/api) diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index 94ccba7540..b9e9d448be 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -29,14 +28,7 @@ void* heap_begin; void* heap_end; -/// IMPLEMENTATION OF Newlib I/O: -#undef stdin -#undef stdout -#undef stderr -// instantiate the Unix standard streams -__FILE* stdin; -__FILE* stdout; -__FILE* stderr; +void* __dso_handle; // stack-protector guard const uintptr_t __stack_chk_guard = (uintptr_t) _STACK_GUARD_VALUE_; @@ -49,6 +41,26 @@ void _init_bss() __builtin_memset(&_BSS_START_, 0, &_BSS_END_ - &_BSS_START_); } +extern void (*__preinit_array_start [])(); +extern void (*__preinit_array_end [])(); +extern void (*__init_array_start [])(); +extern void (*__init_array_end [])(); + +void __libc_init_array() { + size_t count, i; + + count = __preinit_array_end - __preinit_array_start; + for (i = 0; i < count; i++) + __preinit_array_start[i](); + + kprintf("Initializing C constructors \n"); + _init(); + kprintf("Initializing C++ constructors \n"); + count = __init_array_end - __init_array_start; + for (i = 0; i < count; i++) + __init_array_start[i](); +} + void _init_heap(uintptr_t free_mem_begin) { // NOTE: Initialize the heap before exceptions @@ -89,23 +101,29 @@ static void crt_sanity_checks() void _init_c_runtime() { /// initialize newlib I/O - _REENT_INIT_PTR(_REENT); + // _REENT_INIT_PTR(_REENT); // Unix standard streams + + /* stdin = _REENT->_stdin; // stdin == 1 stdout = _REENT->_stdout; // stdout == 2 stderr = _REENT->_stderr; // stderr == 3 + */ /// initialize exceptions before we can run constructors extern char __eh_frame_start[]; // Tell the stack unwinder where exception frames are located extern void __register_frame(void*); - __register_frame(&__eh_frame_start); + kprintf("Not registering frame... \n"); + //__register_frame(&__eh_frame_start); + //kprintf("Registering frame OK \n"); /// init ELF / backtrace functionality extern void _init_elf_parser(); _init_elf_parser(); - + kprintf("Elf parser initialized \n"); crt_sanity_checks(); + kprintf("Sanity checks OK \n"); } // stack-protector @@ -130,25 +148,13 @@ int* __errno_location(void) return &errno; } -#include -int _setjmp(jmp_buf env) -{ - return setjmp(env); -} + // linux strchr variant (NOTE: not completely the same!) void *__rawmemchr (const void *s, int c) { return strchr((const char*) s, c); } -/// assert() interface of ISO POSIX (2003) -void __assert_fail(const char * assertion, const char * file, unsigned int line, const char * function) -{ - if (function) - printf("%s:%u %s: Assertion %s failed.", file, line, function, assertion); - else - printf("%s:%u Assertion %s failed.", file, line, assertion); -} /// sched_yield() causes the calling thread to relinquish the CPU. The /// thread is moved to the end of the queue for its static priority diff --git a/src/kernel/heap.cpp b/src/kernel/heap.cpp index e14d1e02ef..2c65ab5a92 100644 --- a/src/kernel/heap.cpp +++ b/src/kernel/heap.cpp @@ -16,33 +16,17 @@ // limitations under the License. #include - -struct mallinfo { - int arena; /* total space allocated from system */ - int ordblks; /* number of non-inuse chunks */ - int smblks; /* unused -- always zero */ - int hblks; /* number of mmapped regions */ - int hblkhd; /* total space in mmapped regions */ - int usmblks; /* unused -- always zero */ - int fsmblks; /* unused -- always zero */ - int uordblks; /* total allocated space */ - int fordblks; /* total non-inuse space */ - int keepcost; /* top-most, releasable (via malloc_trim) space */ -}; -extern "C" struct mallinfo mallinfo(); -extern "C" void malloc_trim(size_t); extern uintptr_t heap_begin; extern uintptr_t heap_end; uintptr_t OS::heap_usage() noexcept { - struct mallinfo info = mallinfo(); - return info.uordblks; + return ::heap_end; } void OS::heap_trim() noexcept { - malloc_trim(0); + //malloc_trim(0); } uintptr_t OS::heap_max() noexcept diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index a37e987f99..262b5dbe4f 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -51,11 +51,13 @@ extern "C" { extern "C" void abort() { + kprintf("Abooooooring!\n"); panic("abort() called"); } extern "C" void abort_message(const char* fmt, ...) { + kprintf("Aboooooort message %s \n", fmt); char buffer[1024]; va_list list; va_start(list, fmt); diff --git a/src/platform/x86_nano/kernel_start.cpp b/src/platform/x86_nano/kernel_start.cpp index c9460a3af2..62e6a11a57 100644 --- a/src/platform/x86_nano/kernel_start.cpp +++ b/src/platform/x86_nano/kernel_start.cpp @@ -34,8 +34,13 @@ extern "C" { void _init_syscalls(); void __libc_init_array(); uintptr_t _end; + int __libc_start_main(int (*main)(int,char **,char **), int argc, char **argv); + + } + + extern "C" void kernel_start(uintptr_t magic, uintptr_t addr) { @@ -55,20 +60,28 @@ void kernel_start(uintptr_t magic, uintptr_t addr) // Initialize zero-initialized vars _init_bss(); + kprintf("Hello musl worldz!\n"); + // Initialize heap _init_heap(free_mem_begin); + kprintf("Initialized heap\n"); //Initialize stack-unwinder, call global constructors etc. _init_c_runtime(); + kprintf("C++ exceptions initialized\n"); // Initialize system calls _init_syscalls(); //Initialize stdout handlers OS::add_stdout(&OS::default_stdout); + kprintf("stdout added\n"); // Call global ctors - __libc_init_array(); + //__libc_init_array(); + + kprintf("Global ctors NOT called - calling printf\n"); + printf("Hello world %s \n", "of musl"); __platform_init(); diff --git a/src/platform/x86_nano/platform.cpp b/src/platform/x86_nano/platform.cpp index 9ddc427ca0..cf03616181 100644 --- a/src/platform/x86_nano/platform.cpp +++ b/src/platform/x86_nano/platform.cpp @@ -25,6 +25,9 @@ int64_t __arch_time_now() noexcept { } // not supported! void OS::block() {} + +void OS::halt() { asm ("hlt"); } + // default to serial void OS::default_stdout(const char* str, const size_t len) { diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 74bdcc1dce..67178e30f4 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -37,7 +37,7 @@ #endif extern "C" void* get_cpu_esp(); -extern "C" void __libc_init_array(); +//extern "C" void __libc_init_array(); extern uintptr_t heap_begin; extern uintptr_t heap_end; extern uintptr_t _start; @@ -114,7 +114,7 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) // Call global ctors PROFILE("Global constructors"); - __libc_init_array(); + //__libc_init_array(); // BOOT METHOD // PROFILE("Multiboot / legacy"); diff --git a/src/platform/x86_pc/serial1.cpp b/src/platform/x86_pc/serial1.cpp index 02961453df..101b62b116 100644 --- a/src/platform/x86_pc/serial1.cpp +++ b/src/platform/x86_pc/serial1.cpp @@ -32,3 +32,8 @@ void __serial_print(const char* str, size_t len) hw::outb(port, str[i]); } } + +extern "C" +void kprint(const char* c){ + __serial_print1(c); +} diff --git a/vmbuild/CMakeLists.txt b/vmbuild/CMakeLists.txt index 78d69d670d..c4dab9a59c 100644 --- a/vmbuild/CMakeLists.txt +++ b/vmbuild/CMakeLists.txt @@ -11,6 +11,7 @@ set(CMAKE_CXX_FLAGS "-std=c++14 -Wall -Wextra -O2 -g") # TODO: write scripts that automatically find include directories include_directories(. ./../mod/GSL/) + add_executable(vmbuild ${SOURCES}) add_executable(elf_syms ${ELF_SYMS_SOURCES}) diff --git a/vmbuild/vmbuild.cpp b/vmbuild/vmbuild.cpp index 69f023a718..d173ad7c9d 100644 --- a/vmbuild/vmbuild.cpp +++ b/vmbuild/vmbuild.cpp @@ -25,9 +25,9 @@ #include #include #include +#include #include "../api/boot/multiboot.h" -#include "../api/util/elf.h" #define GSL_THROW_ON_CONTRACT_VIOLATION #include "../api/util/elf_binary.hpp" From d807ce985bb4a7caf55e7367f497fed0f2f65509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 11 Jan 2018 10:37:26 +0100 Subject: [PATCH 018/723] vmbuild: Use musl elf.h --- CMakeLists.txt | 4 +++- vmbuild/CMakeLists.txt | 2 +- vmbuild/vmbuild.cpp | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 65f00634fe..9c50710158 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -173,7 +173,9 @@ if(vmbuild) SOURCE_DIR ${INCLUDEOS_ROOT}/vmbuild # Where is project located BINARY_DIR ${INCLUDEOS_ROOT}/vmbuild/build INSTALL_DIR ${BIN} # Where to install - CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= # Pass installation folder + CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= + -DMUSL_INCLUDE=${MUSL_INCLUDE_DIR} # Pass installation folder + DEPENDS PrecompiledLibraries ) endif(vmbuild) diff --git a/vmbuild/CMakeLists.txt b/vmbuild/CMakeLists.txt index c4dab9a59c..bd82c85326 100644 --- a/vmbuild/CMakeLists.txt +++ b/vmbuild/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_CXX_FLAGS "-std=c++14 -Wall -Wextra -O2 -g") # TODO: write scripts that automatically find include directories include_directories(. ./../mod/GSL/) - +include_directories(${MUSL_INCLUDE}/../../) add_executable(vmbuild ${SOURCES}) add_executable(elf_syms ${ELF_SYMS_SOURCES}) diff --git a/vmbuild/vmbuild.cpp b/vmbuild/vmbuild.cpp index d173ad7c9d..58f7a407f5 100644 --- a/vmbuild/vmbuild.cpp +++ b/vmbuild/vmbuild.cpp @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include "../api/boot/multiboot.h" From 3669e83e9f74543487645ada531a24c835277f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 11 Jan 2018 14:31:32 +0100 Subject: [PATCH 019/723] build: Copy elf.h to INSTALL_PREFIX/include --- CMakeLists.txt | 2 +- api/util/elf_binary.hpp | 2 +- cmake/cross_compiled_libraries.txt | 3 +++ vmbuild/CMakeLists.txt | 2 +- vmbuild/vmbuild.cpp | 2 +- 5 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c50710158..657fc9a2ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -174,7 +174,7 @@ if(vmbuild) BINARY_DIR ${INCLUDEOS_ROOT}/vmbuild/build INSTALL_DIR ${BIN} # Where to install CMAKE_ARGS -DCMAKE_INSTALL_PREFIX= - -DMUSL_INCLUDE=${MUSL_INCLUDE_DIR} # Pass installation folder + -DINCLUDE_PATH=${CMAKE_INSTALL_PREFIX} # Pass installation folder DEPENDS PrecompiledLibraries ) endif(vmbuild) diff --git a/api/util/elf_binary.hpp b/api/util/elf_binary.hpp index 4a1fb1138b..670728c96d 100644 --- a/api/util/elf_binary.hpp +++ b/api/util/elf_binary.hpp @@ -18,7 +18,7 @@ #ifndef ELF_BINARY_HPP #define ELF_BINARY_HPP -#include "elf.h" +#include #include #include diff --git a/cmake/cross_compiled_libraries.txt b/cmake/cross_compiled_libraries.txt index 15d34c855c..e08bd59782 100644 --- a/cmake/cross_compiled_libraries.txt +++ b/cmake/cross_compiled_libraries.txt @@ -108,6 +108,9 @@ install(DIRECTORY ${MUSL_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/musl file(GLOB musl_libs ${MUSL_LIB_DIR}/*.a) file(GLOB musl_objs ${MUSL_LIB_DIR}/*.o) +message(STATUS "Installing elf.h into ${CMAKE_INSTALL_PREFIX}/include") +file(COPY ${MUSL_INCLUDE_DIR}/elf.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/) + install(FILES ${musl_libs} ${musl_objs} diff --git a/vmbuild/CMakeLists.txt b/vmbuild/CMakeLists.txt index bd82c85326..e8aa012ee6 100644 --- a/vmbuild/CMakeLists.txt +++ b/vmbuild/CMakeLists.txt @@ -10,7 +10,7 @@ set(CMAKE_CXX_FLAGS "-std=c++14 -Wall -Wextra -O2 -g") # TODO: write scripts that automatically find include directories include_directories(. ./../mod/GSL/) -include_directories(${MUSL_INCLUDE}/../../) +include_directories(${INCLUDE_PATH}/include) add_executable(vmbuild ${SOURCES}) add_executable(elf_syms ${ELF_SYMS_SOURCES}) diff --git a/vmbuild/vmbuild.cpp b/vmbuild/vmbuild.cpp index 58f7a407f5..d173ad7c9d 100644 --- a/vmbuild/vmbuild.cpp +++ b/vmbuild/vmbuild.cpp @@ -25,7 +25,7 @@ #include #include #include -#include +#include #include "../api/boot/multiboot.h" From c9ad0128720b799ba6c4d4c2ddafd08148da87af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 11 Jan 2018 14:32:19 +0100 Subject: [PATCH 020/723] build: Bump clang version to 4 in macOS install --- etc/install_dependencies_macos.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/etc/install_dependencies_macos.sh b/etc/install_dependencies_macos.sh index 534bd2b06d..2fc7544c9b 100755 --- a/etc/install_dependencies_macos.sh +++ b/etc/install_dependencies_macos.sh @@ -16,7 +16,9 @@ INSTALLED_PIP=0 INSTALLED_PIP_PACKAGES=0 INSTALLED_BINUTILS=0 INSTALLED_SYMLINKING=0 -ALL_DEPENDENCIES="llvm38 nasm cmake jq qemu Caskroom/cask/tuntap libmagic" +CLANG_VERSION=4 +BREW_LLVM="llvm@$CLANG_VERSION" +ALL_DEPENDENCIES="$BREW_LLVM nasm cmake jq qemu Caskroom/cask/tuntap libmagic" PIP_MODS="jsonschema psutil filemagic" # NaCl @@ -205,11 +207,14 @@ fi ############################################################ if [[ $INSTALLED_BREW_PACKAGES -eq 1 ]]; then - mkdir -p $INCLUDEOS_BIN - SRC_CC=$(which clang-3.8) + mkdir -p $INCLUDEOS_BIN + + CLANG_PATH=$(brew --prefix $BREW_LLVM) + + SRC_CC=$CLANG_PATH/bin/clang ln -sf $SRC_CC $INCLUDEOS_BIN/gcc - SRC_CXX=$(which clang++-3.8) + SRC_CXX=$CLANG_PATH/bin/clang++ ln -sf $SRC_CXX $INCLUDEOS_BIN/g++ SRC_NASM=$(which nasm) From 0c410dfbf46ddebea1e87bbbdf036dc6372eeb09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 12 Jan 2018 12:49:36 +0100 Subject: [PATCH 021/723] build: Make sure precompiled libraries exists before copying elf.h --- cmake/cross_compiled_libraries.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmake/cross_compiled_libraries.txt b/cmake/cross_compiled_libraries.txt index e08bd59782..42f519ed58 100644 --- a/cmake/cross_compiled_libraries.txt +++ b/cmake/cross_compiled_libraries.txt @@ -108,8 +108,14 @@ install(DIRECTORY ${MUSL_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/musl file(GLOB musl_libs ${MUSL_LIB_DIR}/*.a) file(GLOB musl_objs ${MUSL_LIB_DIR}/*.o) -message(STATUS "Installing elf.h into ${CMAKE_INSTALL_PREFIX}/include") -file(COPY ${MUSL_INCLUDE_DIR}/elf.h DESTINATION ${CMAKE_INSTALL_PREFIX}/include/) +add_custom_command(TARGET PrecompiledLibraries POST_BUILD + COMMAND ${CMAKE_COMMAND} -E echo "Installed elf.h into ${CMAKE_INSTALL_PREFIX}/include" + ) + +ExternalProject_Add_Step(PrecompiledLibraries copy_elf + COMMAND ${CMAKE_COMMAND} -E copy ${MUSL_INCLUDE_DIR}/elf.h ${CMAKE_INSTALL_PREFIX}/include/ + DEPENDEES download + ) install(FILES ${musl_libs} From a04c3b8a0f629e87e1bc2f355c4e82f4b8b2048f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 16 Jan 2018 11:34:07 +0100 Subject: [PATCH 022/723] musl: Compile syscalls as static lib musl_syscalls --- cmake/post.service.cmake | 6 ++++++ src/CMakeLists.txt | 5 ++++- src/musl/CMakeLists.txt | 9 +++++++++ src/musl/common.hpp | 5 +++++ src/musl/futex.cpp | 7 +++++++ 5 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 src/musl/CMakeLists.txt create mode 100644 src/musl/common.hpp create mode 100644 src/musl/futex.cpp diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 87f10c2a69..4b7452f005 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -303,6 +303,10 @@ add_library(libosdeps STATIC IMPORTED) set_target_properties(libosdeps PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(libosdeps PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libosdeps.a) +add_library(musl_syscalls STATIC IMPORTED) +set_target_properties(musl_syscalls PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(musl_syscalls PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libmusl_syscalls.a) + add_library(libcxx STATIC IMPORTED) add_library(cxxabi STATIC IMPORTED) add_library(libunwind STATIC IMPORTED) @@ -457,6 +461,8 @@ target_link_libraries(service libcxx compiler_rt + musl_syscalls + #--whole-archive crtn --no-whole-archive ) # write binary location to known file diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9126d458b1..9b08510b56 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -38,7 +38,7 @@ set(OS_OBJECTS util/memstream.c util/async.cpp util/statman.cpp util/logger.cpp util/sha1.cpp util/syslog_facility.cpp util/syslogd.cpp util/uri.cpp util/percent_encoding.cpp util/tar.cpp util/path_to_regex.cpp util/config.cpp util/autoconf.cpp - crt/c_abi.c crt/ctype_b_loc.c crt/ctype_tolower_loc.c crt/string.c + crt/ctype_b_loc.c crt/ctype_tolower_loc.c crt/string.c crt/quick_exit.cpp crt/cxx_abi.cpp hw/pci_device.cpp hw/ps2.cpp hw/serial.cpp hw/msi.cpp hw/pci_msi.cpp virtio/virtio.cpp virtio/virtio_queue.cpp @@ -78,6 +78,9 @@ endif(WITH_SOLO5) add_subdirectory(drivers) add_subdirectory(plugins) +# Add musl +add_subdirectory(musl) + install(TARGETS os DESTINATION includeos/${ARCH}/lib) diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt new file mode 100644 index 0000000000..ba229f0a77 --- /dev/null +++ b/src/musl/CMakeLists.txt @@ -0,0 +1,9 @@ + +set(MUSL_OBJECTS + futex.cpp +) + +add_library(musl_syscalls STATIC ${MUSL_OBJECTS}) +add_dependencies(musl_syscalls PrecompiledLibraries) + +install(TARGETS musl_syscalls DESTINATION includeos/${ARCH}/lib) diff --git a/src/musl/common.hpp b/src/musl/common.hpp new file mode 100644 index 0000000000..81d46f6e47 --- /dev/null +++ b/src/musl/common.hpp @@ -0,0 +1,5 @@ +#pragma once + +#include + +#define STUB(X) kprintf("%s called\n", X) diff --git a/src/musl/futex.cpp b/src/musl/futex.cpp new file mode 100644 index 0000000000..cbef165067 --- /dev/null +++ b/src/musl/futex.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_futex() { + STUB("Futex"); + return 0; +} From 3f73aadf3734894a56a8d2d1261f559a5d6fe39f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 16 Jan 2018 14:45:58 +0100 Subject: [PATCH 023/723] musl: Stubbed enough to boot a service --- src/kernel/syscalls.cpp | 20 ++++---------------- src/musl/CMakeLists.txt | 6 +++++- src/musl/brk.cpp | 24 ++++++++++++++++++++++++ src/musl/clock_gettime.cpp | 7 +++++++ src/musl/close.cpp | 7 +++++++ src/musl/common.hpp | 4 +++- src/musl/fstat.cpp | 7 +++++++ src/musl/getrlimit.cpp | 7 +++++++ src/musl/gettimeofday.cpp | 7 +++++++ src/musl/ioctl.cpp | 7 +++++++ src/musl/lseek.cpp | 7 +++++++ src/musl/madvise.cpp | 7 +++++++ src/musl/mmap.cpp | 15 +++++++++++++++ src/musl/mremap.cpp | 7 +++++++ src/musl/munmap.cpp | 7 +++++++ src/musl/nanosleep.cpp | 7 +++++++ src/musl/open.cpp | 7 +++++++ src/musl/prlimit64.cpp | 7 +++++++ src/musl/sched_getaffinity.cpp | 7 +++++++ src/musl/sched_yield.cpp | 7 +++++++ src/musl/set_robust_list.cpp | 7 +++++++ src/musl/sysinfo.cpp | 7 +++++++ src/musl/write.cpp | 7 +++++++ src/musl/writev.cpp | 23 +++++++++++++++++++++++ src/platform/x86_pc/kernel_start.cpp | 20 +++----------------- 25 files changed, 203 insertions(+), 35 deletions(-) create mode 100644 src/musl/brk.cpp create mode 100644 src/musl/clock_gettime.cpp create mode 100644 src/musl/close.cpp create mode 100644 src/musl/fstat.cpp create mode 100644 src/musl/getrlimit.cpp create mode 100644 src/musl/gettimeofday.cpp create mode 100644 src/musl/ioctl.cpp create mode 100644 src/musl/lseek.cpp create mode 100644 src/musl/madvise.cpp create mode 100644 src/musl/mmap.cpp create mode 100644 src/musl/mremap.cpp create mode 100644 src/musl/munmap.cpp create mode 100644 src/musl/nanosleep.cpp create mode 100644 src/musl/open.cpp create mode 100644 src/musl/prlimit64.cpp create mode 100644 src/musl/sched_getaffinity.cpp create mode 100644 src/musl/sched_yield.cpp create mode 100644 src/musl/set_robust_list.cpp create mode 100644 src/musl/sysinfo.cpp create mode 100644 src/musl/write.cpp create mode 100644 src/musl/writev.cpp diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index 9e668c1ef7..7003e26536 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -43,10 +43,10 @@ const char* panic_signature = "\x15\x07\t**** PANIC ****"; char* __env[1] {nullptr}; char** environ {__env}; -extern "C" { - uintptr_t heap_begin; - uintptr_t heap_end; -} + + +extern uintptr_t heap_begin; +extern uintptr_t heap_end; extern "C" void abort() { @@ -77,18 +77,6 @@ void _exit(int status) { default_exit(); } -void* sbrk(ptrdiff_t incr) { - /// NOTE: - /// sbrk gets called really early on, before everything else - if (UNLIKELY(heap_end + incr > OS::heap_max())) { - errno = ENOMEM; - return (void*)-1; - } - auto prev_heap_end = heap_end; - heap_end += incr; - return (void*) prev_heap_end; -} - clock_t times(struct tms*) { panic("SYSCALL TIMES Dummy, returning -1"); return -1; diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index ba229f0a77..e9b843bb39 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -1,6 +1,10 @@ set(MUSL_OBJECTS - futex.cpp + futex.cpp close.cpp lseek.cpp ioctl.cpp writev.cpp + write.cpp brk.cpp madvise.cpp mmap.cpp mremap.cpp munmap.cpp + lseek.cpp sched_getaffinity.cpp sysinfo.cpp prlimit64.cpp + getrlimit.cpp sched_yield.cpp set_robust_list.cpp + nanosleep.cpp open.cpp fstat.cpp clock_gettime.cpp gettimeofday.cpp ) add_library(musl_syscalls STATIC ${MUSL_OBJECTS}) diff --git a/src/musl/brk.cpp b/src/musl/brk.cpp new file mode 100644 index 0000000000..38aa54a604 --- /dev/null +++ b/src/musl/brk.cpp @@ -0,0 +1,24 @@ +#include "common.hpp" +#include +#include +#include + +uintptr_t heap_begin; +uintptr_t heap_end; + +extern "C" +int syscall_SYS_brk(void* end_data_segment) +{ + uintptr_t actual = heap_begin + (uintptr_t)end_data_segment; + int res = actual <= OS::heap_max() ? 0 : -1; + + if(res == 0) + { + heap_end = actual; + } + + STRACE("syscall brk(end_data_seg=%p) = %d\n", + end_data_segment, res); + + return res; +} diff --git a/src/musl/clock_gettime.cpp b/src/musl/clock_gettime.cpp new file mode 100644 index 0000000000..63238a3ed4 --- /dev/null +++ b/src/musl/clock_gettime.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_clock_gettime() { + STUB("clock_gettime"); + return 0; +} diff --git a/src/musl/close.cpp b/src/musl/close.cpp new file mode 100644 index 0000000000..3275273470 --- /dev/null +++ b/src/musl/close.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_close() { + STUB("close"); + return 0; +} diff --git a/src/musl/common.hpp b/src/musl/common.hpp index 81d46f6e47..e58f7f443d 100644 --- a/src/musl/common.hpp +++ b/src/musl/common.hpp @@ -2,4 +2,6 @@ #include -#define STUB(X) kprintf("%s called\n", X) +#define STUB(X) kprintf("syscall %s called\n", X) + +#define STRACE(X, ...) kprintf(X, ##__VA_ARGS__) diff --git a/src/musl/fstat.cpp b/src/musl/fstat.cpp new file mode 100644 index 0000000000..c8903de30b --- /dev/null +++ b/src/musl/fstat.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_fstat() { + STUB("fstat"); + return 0; +} diff --git a/src/musl/getrlimit.cpp b/src/musl/getrlimit.cpp new file mode 100644 index 0000000000..84a813ff5b --- /dev/null +++ b/src/musl/getrlimit.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_getrlimit() { + STUB("getrlimit"); + return 0; +} diff --git a/src/musl/gettimeofday.cpp b/src/musl/gettimeofday.cpp new file mode 100644 index 0000000000..5dfa11319a --- /dev/null +++ b/src/musl/gettimeofday.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_gettimeofday() { + STUB("gettimeofday"); + return 0; +} diff --git a/src/musl/ioctl.cpp b/src/musl/ioctl.cpp new file mode 100644 index 0000000000..67cf4d1843 --- /dev/null +++ b/src/musl/ioctl.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_ioctl() { + STUB("ioctl"); + return 0; +} diff --git a/src/musl/lseek.cpp b/src/musl/lseek.cpp new file mode 100644 index 0000000000..ff3bcbec01 --- /dev/null +++ b/src/musl/lseek.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_lseek() { + STUB("lseek"); + return 0; +} diff --git a/src/musl/madvise.cpp b/src/musl/madvise.cpp new file mode 100644 index 0000000000..e29bfbaa86 --- /dev/null +++ b/src/musl/madvise.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_madvise() { + STUB("madvise"); + return 0; +} diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp new file mode 100644 index 0000000000..34b4445497 --- /dev/null +++ b/src/musl/mmap.cpp @@ -0,0 +1,15 @@ +#include "common.hpp" +#include + +extern uintptr_t heap_begin; +static uintptr_t current_pos = 0; +extern "C" +void* syscall_SYS_mmap(void *addr, size_t length, int prot, int flags, + int fd, off_t offset) +{ + uintptr_t res = heap_begin + current_pos; + current_pos += length; + STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d\n", + addr, length, prot, flags, fd, offset); + return (void*)res; +} diff --git a/src/musl/mremap.cpp b/src/musl/mremap.cpp new file mode 100644 index 0000000000..6de60f361b --- /dev/null +++ b/src/musl/mremap.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_mremap() { + STUB("mremap"); + return 0; +} diff --git a/src/musl/munmap.cpp b/src/musl/munmap.cpp new file mode 100644 index 0000000000..837a26be3c --- /dev/null +++ b/src/musl/munmap.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_munmap() { + STUB("munmap"); + return 0; +} diff --git a/src/musl/nanosleep.cpp b/src/musl/nanosleep.cpp new file mode 100644 index 0000000000..d8adeae924 --- /dev/null +++ b/src/musl/nanosleep.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_nanosleep() { + STUB("nanosleep"); + return 0; +} diff --git a/src/musl/open.cpp b/src/musl/open.cpp new file mode 100644 index 0000000000..9eb3d00aaf --- /dev/null +++ b/src/musl/open.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_open() { + STUB("open"); + return 0; +} diff --git a/src/musl/prlimit64.cpp b/src/musl/prlimit64.cpp new file mode 100644 index 0000000000..eaaf85582b --- /dev/null +++ b/src/musl/prlimit64.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_prlimit64() { + STUB("prlimit64"); + return 0; +} diff --git a/src/musl/sched_getaffinity.cpp b/src/musl/sched_getaffinity.cpp new file mode 100644 index 0000000000..c5b13e214a --- /dev/null +++ b/src/musl/sched_getaffinity.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_sched_getaffinity() { + STUB("sched_getaffinity"); + return 0; +} diff --git a/src/musl/sched_yield.cpp b/src/musl/sched_yield.cpp new file mode 100644 index 0000000000..91d3bd0af3 --- /dev/null +++ b/src/musl/sched_yield.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_sched_yield() { + STUB("sched_yield"); + return 0; +} diff --git a/src/musl/set_robust_list.cpp b/src/musl/set_robust_list.cpp new file mode 100644 index 0000000000..d67b3a8580 --- /dev/null +++ b/src/musl/set_robust_list.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_set_robust_list() { + STUB("set_robust_list"); + return 0; +} diff --git a/src/musl/sysinfo.cpp b/src/musl/sysinfo.cpp new file mode 100644 index 0000000000..d644b84d64 --- /dev/null +++ b/src/musl/sysinfo.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_sysinfo() { + STUB("sysinfo"); + return 0; +} diff --git a/src/musl/write.cpp b/src/musl/write.cpp new file mode 100644 index 0000000000..8b69595c0e --- /dev/null +++ b/src/musl/write.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_write() { + STUB("write"); + return 0; +} diff --git a/src/musl/writev.cpp b/src/musl/writev.cpp new file mode 100644 index 0000000000..f0d382e54b --- /dev/null +++ b/src/musl/writev.cpp @@ -0,0 +1,23 @@ +#include "common.hpp" + +#include + +extern "C" +ssize_t syscall_SYS_writev(int fd, const struct iovec *iov, int iovcnt) +{ + if(fd <= 2) + { + ssize_t res = 0; + for(int i = 0; i < iovcnt; i++) + { + auto* text = (const char*)iov[i].iov_base; + auto len = iov[i].iov_len; + kprintf("%.*s", len, text); + res += len; + } + return res; + } + STRACE("syscall writev: fd=%d iov=%p iovcount=%d [base=%p sz=%d]\n", + fd, iov, iovcnt, iov->iov_base, iov->iov_len); + return 0; +} diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 4770a66020..f933c177fa 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -25,11 +25,7 @@ extern "C" { void __init_sanity_checks(); void kernel_sanity_checks(); uintptr_t _multiboot_free_begin(uintptr_t boot_addr); - uintptr_t _move_symbols(uintptr_t loc); - void _init_bss(); - void _init_heap(uintptr_t); - void _init_c_runtime(); - void _init_syscalls(); + //uintptr_t _move_symbols(uintptr_t loc); } extern "C" @@ -51,19 +47,9 @@ void kernel_start(uintptr_t magic, uintptr_t addr) } // Preserve symbols from the ELF binary - free_mem_begin += _move_symbols(free_mem_begin); + //free_mem_begin += _move_symbols(free_mem_begin); - // Initialize zero-initialized vars - _init_bss(); - - // Initialize heap - _init_heap(free_mem_begin); - - // Initialize stack-unwinder, call global constructors etc. - _init_c_runtime(); - - // Initialize system calls - _init_syscalls(); + // TODO: set heap begin // Initialize early OS, platform and devices OS::start(magic, addr); From 170a1be5df309a486e3c496a02e300bbbf931b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 16 Jan 2018 15:34:58 +0100 Subject: [PATCH 024/723] musl: Work in aux vector and init libc --- api/kernel/auxvec.h | 36 +++++++++++++++++++ src/arch/i686/linker.ld | 2 ++ src/arch/x86_64/linker.ld | 2 ++ src/musl/CMakeLists.txt | 1 + src/musl/exit.cpp | 7 ++++ src/musl/exit_group.cpp | 7 ++++ src/musl/mmap2.cpp | 7 ++++ src/musl/poll.cpp | 7 ++++ src/musl/set_tid_address.cpp | 7 ++++ src/platform/x86_pc/kernel_start.cpp | 53 +++++++++++++++++++++++----- 10 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 api/kernel/auxvec.h create mode 100644 src/musl/exit.cpp create mode 100644 src/musl/exit_group.cpp create mode 100644 src/musl/mmap2.cpp create mode 100644 src/musl/poll.cpp create mode 100644 src/musl/set_tid_address.cpp diff --git a/api/kernel/auxvec.h b/api/kernel/auxvec.h new file mode 100644 index 0000000000..e0db2c62ea --- /dev/null +++ b/api/kernel/auxvec.h @@ -0,0 +1,36 @@ +#ifndef KERNEL_AUXVEC_H +#define KERNEL_AUXVEC_H + +// From: https://github.com/torvalds/linux/blob/v3.19/include/uapi/linux/auxvec.h + +/* Symbolic values for the entries in the auxiliary table + put on the initial stack */ +#define AT_NULL 0 /* end of vector */ +#define AT_IGNORE 1 /* entry should be ignored */ +#define AT_EXECFD 2 /* file descriptor of program */ +#define AT_PHDR 3 /* program headers for program */ +#define AT_PHENT 4 /* size of program header entry */ +#define AT_PHNUM 5 /* number of program headers */ +#define AT_PAGESZ 6 /* system page size */ +#define AT_BASE 7 /* base address of interpreter */ +#define AT_FLAGS 8 /* flags */ +#define AT_ENTRY 9 /* entry point of program */ +#define AT_NOTELF 10 /* program is not ELF */ +#define AT_UID 11 /* real uid */ +#define AT_EUID 12 /* effective uid */ +#define AT_GID 13 /* real gid */ +#define AT_EGID 14 /* effective gid */ +#define AT_PLATFORM 15 /* string identifying CPU for optimizations */ +#define AT_HWCAP 16 /* arch dependent hints at CPU capabilities */ +#define AT_CLKTCK 17 /* frequency at which times() increments */ +/* AT_* values 18 through 22 are reserved */ +#define AT_SECURE 23 /* secure mode boolean */ +#define AT_BASE_PLATFORM 24 /* string identifying real platform, may + * differ from AT_PLATFORM. */ +#define AT_RANDOM 25 /* address of 16 random bytes */ +#define AT_HWCAP2 26 /* extension of AT_HWCAP */ + +#define AT_EXECFN 31 /* filename of program */ + + +#endif /* KERNEL_AUXVEC_H */ diff --git a/src/arch/i686/linker.ld b/src/arch/i686/linker.ld index 9a7f108107..86d10822f4 100644 --- a/src/arch/i686/linker.ld +++ b/src/arch/i686/linker.ld @@ -46,7 +46,9 @@ SECTIONS } .fini ALIGN(0x10) : { + _FINI_START_ = .; *(.fini) + _FINI_END_ = .; } /* Global offset-table. For dynamic linking */ diff --git a/src/arch/x86_64/linker.ld b/src/arch/x86_64/linker.ld index c40f0ac27b..a67b45e883 100644 --- a/src/arch/x86_64/linker.ld +++ b/src/arch/x86_64/linker.ld @@ -51,7 +51,9 @@ SECTIONS } .fini ALIGN(0x10) : { + _FINI_START_ = .; *(.fini) + _FINI_END_ = .; } /* Global offset-table. For dynamic linking */ diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index e9b843bb39..b217df7e97 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -5,6 +5,7 @@ set(MUSL_OBJECTS lseek.cpp sched_getaffinity.cpp sysinfo.cpp prlimit64.cpp getrlimit.cpp sched_yield.cpp set_robust_list.cpp nanosleep.cpp open.cpp fstat.cpp clock_gettime.cpp gettimeofday.cpp + poll.cpp mmap2.cpp exit_group.cpp exit.cpp close.cpp set_tid_address.cpp ) add_library(musl_syscalls STATIC ${MUSL_OBJECTS}) diff --git a/src/musl/exit.cpp b/src/musl/exit.cpp new file mode 100644 index 0000000000..812c2ffb5c --- /dev/null +++ b/src/musl/exit.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_exit() { + STUB("exit"); + return 0; +} diff --git a/src/musl/exit_group.cpp b/src/musl/exit_group.cpp new file mode 100644 index 0000000000..b9f5065a6f --- /dev/null +++ b/src/musl/exit_group.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_exit_group() { + STUB("exit_group"); + return 0; +} diff --git a/src/musl/mmap2.cpp b/src/musl/mmap2.cpp new file mode 100644 index 0000000000..c2731f8965 --- /dev/null +++ b/src/musl/mmap2.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_mmap2() { + STUB("mmap2"); + return 0; +} diff --git a/src/musl/poll.cpp b/src/musl/poll.cpp new file mode 100644 index 0000000000..dbbe073b4d --- /dev/null +++ b/src/musl/poll.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_poll() { + STUB("poll"); + return 0; +} diff --git a/src/musl/set_tid_address.cpp b/src/musl/set_tid_address.cpp new file mode 100644 index 0000000000..b07864ad96 --- /dev/null +++ b/src/musl/set_tid_address.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_set_tid_address() { + STUB("set_tid_address"); + return 0; +} diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index f933c177fa..cca3d44b52 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -28,6 +28,42 @@ extern "C" { //uintptr_t _move_symbols(uintptr_t loc); } +int* kernel_main(int, char * *, char * *) { + // Initialize early OS, platform and devices + OS::start(0u,0u); + + // Initialize common subsystems and call Service::start + OS::post_start(); + + // verify certain read-only sections in memory + kernel_sanity_checks(); + + // Starting event loop from here allows us to profile OS::start + OS::event_loop(); +} + +typedef struct +{ + long int a_type; /* Entry type */ + union + { + long int a_val; /* Integer value */ + void *a_ptr; /* Pointer value */ + void (*a_fcn) (void); /* Function pointer value */ + } a_un; +} auxv_t; + +extern "C" +int __libc_start_main(int *(main) (int, char * *, char * *), + int argc, char * * ubp_av, + void (*init) (void), + void (*fini) (void), + void (*rtld_fini) (void), + void (* stack_end)); + +extern "C" void _init(); +extern "C" void _fini(); + extern "C" __attribute__((no_sanitize("all"))) void kernel_start(uintptr_t magic, uintptr_t addr) @@ -51,15 +87,14 @@ void kernel_start(uintptr_t magic, uintptr_t addr) // TODO: set heap begin - // Initialize early OS, platform and devices - OS::start(magic, addr); - - // Initialize common subsystems and call Service::start - OS::post_start(); + extern char _INIT_START_; + extern char _FINI_START_; + void* init_location = &_INIT_START_; + void* fini_location = &_FINI_START_; + void* rtld_fini = nullptr; + void* stack_end = (void*) 0x10000; - // verify certain read-only sections in memory - kernel_sanity_checks(); + __libc_start_main(kernel_main, 0, nullptr, + _init, _fini, [](){}, stack_end); - // Starting event loop from here allows us to profile OS::start - OS::event_loop(); } From 175ee6c4d1b94920d34775a02f927fc2d0d71713 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 18 Jan 2018 16:09:46 +0100 Subject: [PATCH 025/723] Create arguments, auxiliary vec, environment etc. --- api/kernel/auxvec.h | 23 ++++ src/musl/CMakeLists.txt | 2 +- src/musl/fcntl.cpp | 8 ++ src/musl/mmap.cpp | 10 ++ src/musl/mmap2.cpp | 6 -- src/musl/poll.cpp | 10 +- src/platform/x86_pc/kernel_start.cpp | 72 ++++++++++--- src/platform/x86_pc/musl_init.cpp | 155 +++++++++++++++++++++++++++ 8 files changed, 265 insertions(+), 21 deletions(-) create mode 100644 src/musl/fcntl.cpp create mode 100644 src/platform/x86_pc/musl_init.cpp diff --git a/api/kernel/auxvec.h b/api/kernel/auxvec.h index e0db2c62ea..a73ba16008 100644 --- a/api/kernel/auxvec.h +++ b/api/kernel/auxvec.h @@ -32,5 +32,28 @@ #define AT_EXECFN 31 /* filename of program */ +typedef struct +{ + long int a_type; /* Entry type */ + union + { + long int a_val; /* Integer value */ + const void *a_ptr; /* Pointer value */ + void (*a_fcn) (void); /* Function pointer value */ + } a_un; + + void set_long(long t, long v) { + a_type = t; + a_un.a_val = v; + } + void set_ptr(long t, const void* ptr) { + a_type = t; + a_un.a_ptr = ptr; + } + void set_func(long t, void (*fn)()) { + a_type = t; + a_un.a_fcn = fn; + } +} auxv_t; #endif /* KERNEL_AUXVEC_H */ diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index b217df7e97..6602b237fc 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -1,6 +1,6 @@ set(MUSL_OBJECTS - futex.cpp close.cpp lseek.cpp ioctl.cpp writev.cpp + futex.cpp close.cpp fcntl.cpp lseek.cpp ioctl.cpp writev.cpp write.cpp brk.cpp madvise.cpp mmap.cpp mremap.cpp munmap.cpp lseek.cpp sched_getaffinity.cpp sysinfo.cpp prlimit64.cpp getrlimit.cpp sched_yield.cpp set_robust_list.cpp diff --git a/src/musl/fcntl.cpp b/src/musl/fcntl.cpp new file mode 100644 index 0000000000..764a7a0403 --- /dev/null +++ b/src/musl/fcntl.cpp @@ -0,0 +1,8 @@ +#include "common.hpp" + +extern "C" +int syscall_SYS_fcntl(int fd, int cmd, ... /* arg */ ) +{ + STRACE("fcntl(%d, %x) = 0\n", fd, cmd); + return 0; +} diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 34b4445497..db2aeec27e 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -13,3 +13,13 @@ void* syscall_SYS_mmap(void *addr, size_t length, int prot, int flags, addr, length, prot, flags, fd, offset); return (void*)res; } + +extern "C" +void* syscall_SYS_mmap2(void *addr, size_t length, int prot, + int flags, int fd, off_t offset) { + uintptr_t res = heap_begin + current_pos; + current_pos += length; + STRACE("syscall mmap2: addr=%p len=%u prot=%d fl=%d fd=%d off=%d\n", + addr, length, prot, flags, fd, offset); + return (void*)res; +} diff --git a/src/musl/mmap2.cpp b/src/musl/mmap2.cpp index c2731f8965..7a1adb1edf 100644 --- a/src/musl/mmap2.cpp +++ b/src/musl/mmap2.cpp @@ -1,7 +1 @@ #include "common.hpp" - -extern "C" -long syscall_SYS_mmap2() { - STUB("mmap2"); - return 0; -} diff --git a/src/musl/poll.cpp b/src/musl/poll.cpp index dbbe073b4d..eb68e6219a 100644 --- a/src/musl/poll.cpp +++ b/src/musl/poll.cpp @@ -1,7 +1,13 @@ #include "common.hpp" +#include extern "C" -long syscall_SYS_poll() { +long syscall_SYS_poll(struct pollfd *fds, nfds_t nfds, int timeout) +{ STUB("poll"); - return 0; + for (int i = 0; i < nfds; i++) + { + fds[i].revents = fds[i].events; + } + return nfds; } diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index cca3d44b52..6abc1bd068 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -19,6 +19,7 @@ #include #include #include +#include "musl_init.cpp" extern "C" { void __init_serial1(); @@ -28,7 +29,21 @@ extern "C" { //uintptr_t _move_symbols(uintptr_t loc); } -int* kernel_main(int, char * *, char * *) { +extern uintptr_t heap_begin; +extern uintptr_t heap_end; + +void _init_heap(uintptr_t free_mem_begin) +{ + #define HEAP_ALIGNMENT 4095 + // NOTE: Initialize the heap before exceptions + // cache-align heap, because its not aligned + heap_begin = free_mem_begin + HEAP_ALIGNMENT; + heap_begin = ((uintptr_t)heap_begin & ~HEAP_ALIGNMENT); + heap_end = heap_begin; +} + +int kernel_main(int, char * *, char * *) { + kprintf("kernel_main() !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"); // Initialize early OS, platform and devices OS::start(0u,0u); @@ -42,16 +57,10 @@ int* kernel_main(int, char * *, char * *) { OS::event_loop(); } -typedef struct +void __init_tls(size_t* p) { - long int a_type; /* Entry type */ - union - { - long int a_val; /* Integer value */ - void *a_ptr; /* Pointer value */ - void (*a_fcn) (void); /* Function pointer value */ - } a_un; -} auxv_t; + kprintf("init_tls(%p)\n", p); +} extern "C" int __libc_start_main(int *(main) (int, char * *, char * *), @@ -63,6 +72,8 @@ int __libc_start_main(int *(main) (int, char * *, char * *), extern "C" void _init(); extern "C" void _fini(); +#include +#include extern "C" __attribute__((no_sanitize("all"))) @@ -86,6 +97,7 @@ void kernel_start(uintptr_t magic, uintptr_t addr) //free_mem_begin += _move_symbols(free_mem_begin); // TODO: set heap begin + _init_heap(free_mem_begin); extern char _INIT_START_; extern char _FINI_START_; @@ -94,7 +106,43 @@ void kernel_start(uintptr_t magic, uintptr_t addr) void* rtld_fini = nullptr; void* stack_end = (void*) 0x10000; - __libc_start_main(kernel_main, 0, nullptr, - _init, _fini, [](){}, stack_end); + auxv_t aux[38]; + int i = 0; + aux[i++].set_long(AT_PAGESZ, 4096); + aux[i++].set_long(AT_CLKTCK, 100); + + aux[i++].set_long(AT_PHENT, 32); + aux[i++].set_ptr(AT_PHDR, 0x0); + aux[i++].set_long(AT_PHNUM, 0); + aux[i++].set_ptr(AT_BASE, nullptr); + aux[i++].set_long(AT_FLAGS, 0x0); + aux[i++].set_ptr(AT_ENTRY, (void*) &kernel_main); + aux[i++].set_long(AT_HWCAP, 0); + + aux[i++].set_long(AT_UID, 0); + aux[i++].set_long(AT_EUID, 0); + aux[i++].set_long(AT_GID, 0); + aux[i++].set_long(AT_EGID, 0); + aux[i++].set_long(AT_SECURE, 1); + + const char* plat = "x86_64"; + aux[i++].set_ptr(AT_PLATFORM, plat); + + aux[i++].set_long(AT_NULL, 0); + + std::array argv; + // arguments to "program" + argv[0] = (char*) Service::name(); + argv[1] = 0x0; + int argc = 1; + + // "environment" variables + argv[2] = "BLARGH=0"; + argv[3] = 0x0; + + memcpy(&argv[4], aux, sizeof(auxv_t) * 38); + + // ubp_av = argv, irrelevant when argc = 0 + hallo__libc_start_main(kernel_main, argc, argv.data()); } diff --git a/src/platform/x86_pc/musl_init.cpp b/src/platform/x86_pc/musl_init.cpp new file mode 100644 index 0000000000..6cb93603d2 --- /dev/null +++ b/src/platform/x86_pc/musl_init.cpp @@ -0,0 +1,155 @@ +#include +#include +#include +#include +extern "C" void panic(const char*); + +#ifndef LIBC_H +#define LIBC_H + +#include +#include +#include + +struct __locale_map; + +struct __locale_struct { + const struct __locale_map *volatile cat[6]; +}; + +struct tls_module { + struct tls_module *next; + void *image; + size_t len, size, align, offset; +}; + +struct __libc { + int can_do_threads; + int threaded; + int secure; + volatile int threads_minus_1; + size_t *auxv; + struct tls_module *tls_head; + size_t tls_size, tls_align, tls_cnt; + size_t page_size; + struct __locale_struct global_locale; +}; + +#ifndef PAGE_SIZE +#define PAGE_SIZE libc.page_size +#endif + +#ifdef __PIC__ +#define ATTR_LIBC_VISIBILITY __attribute__((visibility("hidden"))) +#else +#define ATTR_LIBC_VISIBILITY +#endif + +extern struct __libc __libc ATTR_LIBC_VISIBILITY; +#define libc __libc + +extern size_t __hwcap ATTR_LIBC_VISIBILITY; +extern size_t __sysinfo ATTR_LIBC_VISIBILITY; +extern char *__progname, *__progname_full; + +/* Designed to avoid any overhead in non-threaded processes */ +void __lock(volatile int *) ATTR_LIBC_VISIBILITY; +void __unlock(volatile int *) ATTR_LIBC_VISIBILITY; +int __lockfile(FILE *) ATTR_LIBC_VISIBILITY; +void __unlockfile(FILE *) ATTR_LIBC_VISIBILITY; +#define LOCK(x) __lock(x) +#define UNLOCK(x) __unlock(x) + +void __synccall(void (*)(void *), void *); +int __setxid(int, int, int, int); + +extern char **__environ; + +#undef weak_alias +#define weak_alias(old, new) \ + extern __typeof(old) new __attribute__((weak, alias(#old))) + +#undef LFS64_2 +#define LFS64_2(x, y) weak_alias(x, y) + +#undef LFS64 +#define LFS64(x) LFS64_2(x, x##64) + +#endif + + +void __init_tls(size_t *); + +extern "C" void _init(); + +__attribute__((__weak__, __visibility__("hidden"))) +extern void (*const __init_array_start)(void), (*const __init_array_end)(void); + +void __init_ssp(void*) {} + +#define AUX_CNT 38 + +void hallo__init_libc(char **envp, char *pn) +{ + kprintf("__init_libc(%p, %p)\n", envp, pn); + size_t i, *auxv, aux[AUX_CNT] = { 0 }; + __environ = envp; + for (i=0; envp[i]; i++); + libc.auxv = auxv = (size_t *)(envp+i+1); + kprintf("aux: %p\n", auxv); + + for (i=0; auxv[i]; i+=2) if (auxv[i] Date: Sun, 21 Jan 2018 21:15:05 +0100 Subject: [PATCH 026/723] boot/vmbuild: add flag for debugging with GDB --- api/debug | 21 ++++++++++++++------- cmake/cross_compiled_libraries.txt | 1 + etc/boot | 10 ++++++++-- etc/debug/service.gdb | 7 +++++-- vmrunner/vmrunner.py | 12 ++++++++---- 5 files changed, 36 insertions(+), 15 deletions(-) diff --git a/api/debug b/api/debug index 8648e75fbe..32940bdf33 100644 --- a/api/debug +++ b/api/debug @@ -7,9 +7,9 @@ // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -22,14 +22,21 @@ #if defined(DEBUG) && ! defined(NO_DEBUG) #define debug(X,...) printf(X,##__VA_ARGS__); -#else -#define debug(X,...) +#else +#define debug(X,...) +#endif + +#if defined(__x86_64__) || defined(__i386__) +#define GDB_ENTRY { printf("Waiting for GDB to set %%eax to 1..\n"); \ + asm("mov $0,%eax ; 0: ; test %eax,%eax ; jz 0b"); } +#else +#define GDB_ENTRY #endif + #if defined(DEBUG2) && ! defined(NO_DEBUG) #define debug2(X,...) printf(X,##__VA_ARGS__); -#else -#define debug2(X,...) +#else +#define debug2(X,...) #endif - #endif diff --git a/cmake/cross_compiled_libraries.txt b/cmake/cross_compiled_libraries.txt index 42f519ed58..89f6586ac1 100644 --- a/cmake/cross_compiled_libraries.txt +++ b/cmake/cross_compiled_libraries.txt @@ -113,6 +113,7 @@ add_custom_command(TARGET PrecompiledLibraries POST_BUILD ) ExternalProject_Add_Step(PrecompiledLibraries copy_elf + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_INSTALL_PREFIX}/include/ COMMAND ${CMAKE_COMMAND} -E copy ${MUSL_INCLUDE_DIR}/elf.h ${CMAKE_INSTALL_PREFIX}/include/ DEPENDEES download ) diff --git a/etc/boot b/etc/boot index 107d922427..22bbd27839 100755 --- a/etc/boot +++ b/etc/boot @@ -57,6 +57,9 @@ parser.add_argument("-j", "--config", dest="config", type = str, metavar = "PATH parser.add_argument('vm_location', action="store", type = str, help="Location of the IncludeOS service binary, image or source") +parser.add_argument("-d", "--debug", dest="debug", action="store_true", + help="Start hypervisor in debug mode if available") + parser.add_argument('vmargs', nargs='*', help="Arguments to pass on to the VM start / main") args = parser.parse_args() @@ -126,6 +129,9 @@ if args.clean: print INFO, "Cleaning build" vm.clean() +if (args.debug): + print INFO, "Booting in debug mode" + if args.bridge: print INFO, "Creating bridge" subprocess.call(INCLUDEOS_PREFIX + "/includeos/scripts/create_bridge.sh", shell=True) @@ -161,6 +167,6 @@ if (args.grub or args.grub_reuse): if (args.build): exit(0); if (not has_bootloader): - vm.boot(timeout = None, multiboot = True, kernel_args = " ".join(args.vmargs), image_name = image_name) + vm.boot(timeout = None, multiboot = True, debug = args.debug, kernel_args = " ".join(args.vmargs), image_name = image_name) else: - vm.boot(timeout = None, multiboot = False, kernel_args = None, image_name = image_name) + vm.boot(timeout = None, multiboot = False, debug = args.debug, kernel_args = None, image_name = image_name) diff --git a/etc/debug/service.gdb b/etc/debug/service.gdb index f9c7293aaf..05c7e8973a 100644 --- a/etc/debug/service.gdb +++ b/etc/debug/service.gdb @@ -1,6 +1,9 @@ -break _start -break OS::start +set architecture i386:x86-64 + break Service::start break main + set non-stop off target remote localhost:1234 +set $eax=1 +continue diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 987004fad4..288852a33e 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -277,7 +277,7 @@ def start_process(self, cmdlist): def get_final_output(self): return self._proc.communicate() - def boot(self, multiboot, kernel_args = "", image_name = None): + def boot(self, multiboot, debug = False, kernel_args = "", image_name = None): self._stopped = False info ("Booting with multiboot:", multiboot, "kernel_args: ", kernel_args, "image_name:", image_name) @@ -295,6 +295,10 @@ def boot(self, multiboot, kernel_args = "", image_name = None): disk_args = [] + debug_args = [] + if debug: + debug_args = ["-s"] + # multiboot - e.g. boot with '-kernel' and no bootloader if multiboot: @@ -385,7 +389,7 @@ def boot(self, multiboot, kernel_args = "", image_name = None): command += kernel_args - command += disk_args + net_args + mem_arg + vga_arg + mod_args + command += disk_args + debug_args + net_args + mem_arg + vga_arg + mod_args #command_str = " ".join(command) #command_str.encode('ascii','ignore') @@ -689,7 +693,7 @@ def trigger_event(self, line): # Boot the VM and start reading output. This is the main event loop. - def boot(self, timeout = 60, multiboot = True, kernel_args = "booted with vmrunner", image_name = None): + def boot(self, timeout = 60, multiboot = True, debug = False, kernel_args = "booted with vmrunner", image_name = None): info ("VM boot, timeout: ", timeout, "multiboot: ", multiboot, "Kernel_args: ", kernel_args, "image_name: ", image_name) # This might be a reboot self._exit_status = None @@ -704,7 +708,7 @@ def boot(self, timeout = 60, multiboot = True, kernel_args = "booted with vmrunn # Boot via hypervisor try: - self._hyper.boot(multiboot, kernel_args, image_name) + self._hyper.boot(multiboot, debug, kernel_args, image_name) except Exception as err: print color.WARNING("Exception raised while booting: ") print_exception() From bcd739e9e388bc43c16def221e2e3cdda727cf3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 23 Jan 2018 11:30:40 +0100 Subject: [PATCH 027/723] tcp: Added helper function fits(seq) to Read_buffer --- api/net/tcp/read_buffer.hpp | 11 +++++++++++ src/net/tcp/read_buffer.cpp | 6 ++++++ test/net/unit/tcp_read_buffer_test.cpp | 26 ++++++++++++++++++++++++++ 3 files changed, 43 insertions(+) diff --git a/api/net/tcp/read_buffer.hpp b/api/net/tcp/read_buffer.hpp index dabb5099bd..60e69ea01d 100644 --- a/api/net/tcp/read_buffer.hpp +++ b/api/net/tcp/read_buffer.hpp @@ -56,6 +56,17 @@ class Read_buffer { */ size_t insert(const seq_t seq, const uint8_t* data, size_t len, bool push = false); + /** + * @brief Returns the amount of the bytes that fits in the buffer + * when starting from "seq". + * If the seq is before the start of the buffer, 0 is returned. + * + * @param[in] seq The sequence number + * + * @return Amount of bytes that fits in the buffer starting from seq + */ + size_t fits(const seq_t seq) const; + /** * @brief Exposes the internal buffer * diff --git a/src/net/tcp/read_buffer.cpp b/src/net/tcp/read_buffer.cpp index 3773f2f227..bd0b360b0f 100644 --- a/src/net/tcp/read_buffer.cpp +++ b/src/net/tcp/read_buffer.cpp @@ -55,6 +55,12 @@ size_t Read_buffer::insert(const seq_t seq, const uint8_t* data, size_t len, boo return len; } +size_t Read_buffer::fits(const seq_t seq) const +{ + const auto rel = (seq - start); + return (rel < capacity()) ? (capacity() - rel) : 0; +} + void Read_buffer::reset(const seq_t seq) { this->reset(seq, buf->capacity()); diff --git a/test/net/unit/tcp_read_buffer_test.cpp b/test/net/unit/tcp_read_buffer_test.cpp index acfd74b2c8..d467b47e81 100644 --- a/test/net/unit/tcp_read_buffer_test.cpp +++ b/test/net/unit/tcp_read_buffer_test.cpp @@ -169,3 +169,29 @@ CASE("Reseting the buffer") buf.reset(SEQ, BUFSZ/2); EXPECT(buf.buffer()->data() != data); } + +#include +CASE("fits()") +{ + using namespace net::tcp; + const size_t BUFSZ = 1024; + seq_t seq = 1000; + + std::unique_ptr buf; + + buf.reset(new Read_buffer(BUFSZ, seq)); + + EXPECT(buf->fits(1000) == BUFSZ - (1000 - seq)); + EXPECT(buf->fits(1200) == BUFSZ - (1200 - seq)); + EXPECT(buf->fits(900) == 0); + EXPECT(buf->fits(seq + BUFSZ) == 0); + + const uint32_t MAX_UINT = std::numeric_limits::max(); + + seq = MAX_UINT - 500; + buf.reset(new Read_buffer(BUFSZ, seq)); + EXPECT(buf->fits(seq) == BUFSZ); + EXPECT(buf->fits(seq + 500) == BUFSZ - 500); + EXPECT(buf->fits(seq + 1000) == BUFSZ - 1000); + EXPECT(buf->fits(4000) == 0); +} From 7356633927dc23d5cc77066c789d02a54e5fb479 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 18:15:08 +0100 Subject: [PATCH 028/723] util/elf_binary: don't throw from constructor --- api/util/elf_binary.hpp | 2 +- cmake/cross_compiled_libraries.txt | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/util/elf_binary.hpp b/api/util/elf_binary.hpp index 670728c96d..7f452a5d00 100644 --- a/api/util/elf_binary.hpp +++ b/api/util/elf_binary.hpp @@ -53,7 +53,7 @@ class Elf_binary { Elf_binary(Span data) : data_{data} { - validate(); + Expects(is_ELF()); } const Elf_header& elf_header() const; diff --git a/cmake/cross_compiled_libraries.txt b/cmake/cross_compiled_libraries.txt index 89f6586ac1..f01fb7005c 100644 --- a/cmake/cross_compiled_libraries.txt +++ b/cmake/cross_compiled_libraries.txt @@ -95,15 +95,15 @@ add_library(libm STATIC IMPORTED) set_target_properties(libm PROPERTIES IMPORTED_LOCATION ${MUSL_LIB_DIR}/libm.a) add_dependencies(libm PrecompiledLibraries) -#set(CRTEND ${PRECOMPILED_DIR}/crt/crtend.o) -#set(CRTBEGIN ${PRECOMPILED_DIR}/crt/crtbegin.o) +set(CRTEND ${PRECOMPILED_DIR}/crt/crtend.o) +set(CRTBEGIN ${PRECOMPILED_DIR}/crt/crtbegin.o) # installation instructions install(DIRECTORY ${LIBCXX_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/libcxx) install(DIRECTORY ${MUSL_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/musl) -#install(FILES ${CRTEND} ${CRTBEGIN} DESTINATION includeos/${ARCH}/lib) +install(FILES ${CRTEND} ${CRTBEGIN} DESTINATION includeos/${ARCH}/lib) file(GLOB musl_libs ${MUSL_LIB_DIR}/*.a) file(GLOB musl_objs ${MUSL_LIB_DIR}/*.o) From fd732afcc2f8ab8f31ee2512c5792ff4d2fab4c0 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 18:28:27 +0100 Subject: [PATCH 029/723] bundle: functioning integration with LLVM libunwind --- cmake/cross_compiled_libraries.txt | 9 ++--- cmake/post.service.cmake | 58 +++++++++++++----------------- etc/build_llvm.sh | 28 +++++++-------- etc/create_binary_bundle.sh | 35 ++++++++++++------ 4 files changed, 67 insertions(+), 63 deletions(-) diff --git a/cmake/cross_compiled_libraries.txt b/cmake/cross_compiled_libraries.txt index f01fb7005c..57edf57d85 100644 --- a/cmake/cross_compiled_libraries.txt +++ b/cmake/cross_compiled_libraries.txt @@ -71,6 +71,8 @@ set(PRECOMPILED_DIR ${CMAKE_CURRENT_BINARY_DIR}/precompiled/src/PrecompiledLibra set(LIBCXX_INCLUDE_DIR ${PRECOMPILED_DIR}/libcxx/include/) set(LIBCXX_LIB_DIR ${PRECOMPILED_DIR}/libcxx/) +set(LIBUNWIND_INCLUDE_DIR ${PRECOMPILED_DIR}/libunwind/include/) +set(LIBUNWIND_LIB_DIR ${PRECOMPILED_DIR}/libunwind/) add_library(libcxx STATIC IMPORTED) add_library(libcxxabi STATIC IMPORTED) @@ -80,13 +82,11 @@ add_dependencies(libcxx PrecompiledLibraries) add_dependencies(libcxxabi PrecompiledLibraries) set_target_properties(libcxx PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libc++.a) set_target_properties(libcxxabi PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libc++abi.a) -set_target_properties(libunwind PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libunwind.a) +set_target_properties(libunwind PROPERTIES IMPORTED_LOCATION ${LIBUNWIND_LIB_DIR}/libunwind.a) set(MUSL_INCLUDE_DIR ${PRECOMPILED_DIR}/musl/include/) set(MUSL_LIB_DIR ${PRECOMPILED_DIR}/musl/lib) -#set(LIBGCC_LIB_DIR ${PRECOMPILED_DIR}/libgcc/) - add_library(libc STATIC IMPORTED) set_target_properties(libc PROPERTIES IMPORTED_LOCATION ${MUSL_LIB_DIR}/libc.a) add_dependencies(libc PrecompiledLibraries) @@ -100,6 +100,7 @@ set(CRTBEGIN ${PRECOMPILED_DIR}/crt/crtbegin.o) # installation instructions install(DIRECTORY ${LIBCXX_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/libcxx) +install(DIRECTORY ${LIBUNWIND_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/libunwind) install(DIRECTORY ${MUSL_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/musl) @@ -123,5 +124,5 @@ install(FILES ${musl_objs} ${LIBCXX_LIB_DIR}/libc++.a ${LIBCXX_LIB_DIR}/libc++abi.a - ${LIBCXX_LIB_DIR}/libunwind.a + ${LIBUNWIND_LIB_DIR}/libunwind.a DESTINATION includeos/${ARCH}/lib) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 4b7452f005..2e94d62ed7 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -8,6 +8,7 @@ if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) endif() set(INSTALL_LOC $ENV{INCLUDEOS_PREFIX}/includeos) +set(INSTALL_LOC_LIBGCC $ENV{INCLUDEOS_PREFIX}/includeos) message(STATUS "Target CPU architecture ${ARCH}") set(TRIPLE "${ARCH}-pc-linux-elf") @@ -33,18 +34,10 @@ endif() enable_language(ASM_NASM) -# Find compiler-rt (LLVM's libgcc alternative) -execute_process(COMMAND ${CMAKE_CXX_COMPILER} - -nostdlib -rtlib=compiler-rt -print-libgcc-file-name -target ${TRIPLE} - OUTPUT_VARIABLE compiler_rt_path OUTPUT_STRIP_TRAILING_WHITESPACE) - -message(STATUS "Using compiler-rt at ${compiler_rt_path}") -#set(compiler_rt_path "/usr/lib/llvm-4.0/bin/../lib/clang/4.0.0/lib/linux/libclang_rt.builtins-i686.a") - # Various global defines # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures # * _GNU_SOURCE enables POSIX-extensions in newlib, such as strnlen. ("everything newlib has", ref. cdefs.h) -set(CAPABS "${CAPABS} --rtlib=compiler-rt -fstack-protector-strong -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE -DSERVICE=\"\\\"${BINARY}\\\"\" -DSERVICE_NAME=\"\\\"${SERVICE_NAME}\\\"\"") +set(CAPABS "${CAPABS} -fstack-protector-strong -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE -DSERVICE=\"\\\"${BINARY}\\\"\" -DSERVICE_NAME=\"\\\"${SERVICE_NAME}\\\"\"") set(WARNS "-Wall -Wextra") #-pedantic # Compiler optimization @@ -224,7 +217,7 @@ include_directories(${LOCAL_INCLUDES}) include_directories(${INSTALL_LOC}/api/posix) include_directories(${INSTALL_LOC}/${ARCH}/include/libcxx) include_directories(${INSTALL_LOC}/${ARCH}/include/musl) - +include_directories(${INSTALL_LOC}/${ARCH}/include/libunwind) if ("${PLATFORM}" STREQUAL "x86_solo5") include_directories(${INSTALL_LOC}/${ARCH}/include/solo5) endif() @@ -261,16 +254,16 @@ if ("${PLATFORM}" STREQUAL "x86_solo5") set(PRE_BSS_SIZE "--defsym PRE_BSS_AREA=0x200000") endif() -set(LDFLAGS "-nostdlib -melf_${ELF} -N --eh-frame-hdr ${STRIP_LV} --script=${INSTALL_LOC}/${ARCH}/linker.ld ${PRE_BSS_SIZE} ") #${INSTALL_LOC}/${ARCH}/lib/crtbegin.o") +set(LDFLAGS "-nostdlib -melf_${ELF} --eh-frame-hdr ${STRIP_LV} --script=${INSTALL_LOC}/${ARCH}/linker.ld ${PRE_BSS_SIZE} ${INSTALL_LOC}/${ARCH}/lib/crtbegin.o") set_target_properties(service PROPERTIES LINK_FLAGS "${LDFLAGS}") +set(CRTN "${INSTALL_LOC}/${ARCH}/lib/crtn.o") +set(CRTEND "${INSTALL_LOC}/${ARCH}/lib/crtend.o") +set(CRTI "${INSTALL_LOC}/${ARCH}/lib/crti.o") -add_library(crti STATIC IMPORTED) -set_target_properties(crti PROPERTIES LINKER_LANGUAGE CXX) -set_target_properties(crti PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libcrti.a) - -target_link_libraries(service --whole-archive crti --no-whole-archive) +target_link_libraries(service ${CRTI}) +target_link_libraries(service ${CRT1}) add_library(libos STATIC IMPORTED) set_target_properties(libos PROPERTIES LINKER_LANGUAGE CXX) @@ -310,17 +303,13 @@ set_target_properties(musl_syscalls PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/ add_library(libcxx STATIC IMPORTED) add_library(cxxabi STATIC IMPORTED) add_library(libunwind STATIC IMPORTED) -add_library(compiler_rt STATIC IMPORTED) + set_target_properties(libcxx PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(libcxx PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libc++.a) set_target_properties(cxxabi PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(cxxabi PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libc++abi.a) set_target_properties(libunwind PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(libunwind PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libunwind.a) -set_target_properties(compiler_rt PROPERTIES LINKER_LANGUAGE CXX) -set_target_properties(compiler_rt PROPERTIES IMPORTED_LOCATION ${compiler_rt_path}) - - add_library(libc STATIC IMPORTED) set_target_properties(libc PROPERTIES LINKER_LANGUAGE C) @@ -418,9 +407,10 @@ if(TARFILE) target_link_libraries(service --whole-archive tarfile --no-whole-archive) endif(TARFILE) -add_library(crtn STATIC IMPORTED) -set_target_properties(crtn PROPERTIES LINKER_LANGUAGE CXX) -set_target_properties(crtn PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libcrtn.a) +#add_library(crtn STATIC IMPORTED) +#set_target_properties(crtn PROPERTIES LINKER_LANGUAGE CXX) +#set_target_properties(crtn PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libcrtn.a) + if ("${PLATFORM}" STREQUAL "x86_solo5") target_link_libraries(service solo5 --whole-archive crtn --no-whole-archive) @@ -459,11 +449,11 @@ target_link_libraries(service libpthread libunwind libcxx - compiler_rt musl_syscalls - #--whole-archive crtn --no-whole-archive + ${CRTEND} + ${CRTN} ) # write binary location to known file file(WRITE ${CMAKE_BINARY_DIR}/binary.txt ${BINARY}) @@ -473,13 +463,15 @@ if (debug) set(STRIP_LV /bin/true) endif() -add_custom_target( - pruned_elf_symbols ALL - COMMAND ${INSTALL_LOC}/bin/elf_syms ${BINARY} - COMMAND ${CMAKE_OBJCOPY} --update-section .elf_symbols=_elf_symbols.bin ${BINARY} ${BINARY} - COMMAND ${STRIP_LV} - DEPENDS service -) +if (NOT debug) + add_custom_target( + pruned_elf_symbols ALL + COMMAND ${INSTALL_LOC}/bin/elf_syms ${BINARY} + COMMAND ${CMAKE_OBJCOPY} --update-section .elf_symbols=_elf_symbols.bin ${BINARY} ${BINARY} + COMMAND ${STRIP_LV} + DEPENDS service + ) +endif() # create .img files too automatically add_custom_target( diff --git a/etc/build_llvm.sh b/etc/build_llvm.sh index c1596dc552..d37cddd826 100755 --- a/etc/build_llvm.sh +++ b/etc/build_llvm.sh @@ -4,21 +4,22 @@ # Download, configure, compile and install llvm ARCH=${ARCH:-x86_64} # CPU architecture. Alternatively x86_64 TARGET=$ARCH-elf # Configure target based on arch. Always ELF. +BUILD_DIR=${BUILD_DIR:-~/IncludeOS_build/build_llvm} musl_inc=$TEMP_INSTALL_DIR/$TARGET/include # path for newlib headers IncludeOS_posix=$INCLUDEOS_SRC/api/posix libcxx_inc=$BUILD_DIR/llvm/projects/libcxx/include libcxxabi_inc=$BUILD_DIR/llvm/projects/libcxxabi/include +threading=${INCLUDEOS_THREADING:-OFF} # Install dependencies sudo apt-get install -y ninja-build zlib1g-dev libtinfo-dev cd $BUILD_DIR - download_llvm=${download_llvm:-"1"} # This should be more dynamic - if [ ! -z $download_llvm ]; then - # Clone LLVM + # Clone LLVM + echo "Downloading LLVM" git clone -b $llvm_branch https://github.com/llvm-mirror/llvm.git || true #svn co http://llvm.org/svn/llvm-project/llvm/tags/$LLVM_TAG llvm @@ -64,7 +65,7 @@ fi TRIPLE=$ARCH-pc-linux-elf -CXX_FLAGS="-std=c++14 -msse3 -mfpmath=sse -D_LIBCPP_HAS_MUSL_LIBC" +CXX_FLAGS="-std=c++14 -msse3 -mfpmath=sse -nostdlibinc -D_LIBCPP_HAS_MUSL_LIBC" # CMAKE configure step # @@ -75,37 +76,32 @@ CXX_FLAGS="-std=c++14 -msse3 -mfpmath=sse -D_LIBCPP_HAS_MUSL_LIBC" echo "Building LLVM for $TRIPLE" - # -DCMAKE_C_COMPILER=clang-$clang_version \ - # -DCMAKE_CXX_COMPILER=clang++-$clang_version \ cmake -GNinja $OPTS \ -DCMAKE_CXX_FLAGS="$CXX_FLAGS -I$IncludeOS_posix -I$libcxxabi_inc -I$libcxx_inc -I$musl_inc " $BUILD_DIR/llvm \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DCMAKE_INSTALL_PREFIX=$BUILD_DIR/IncludeOS_TEMP_install \ -DBUILD_SHARED_LIBS=OFF \ -DTARGET_TRIPLE=$TRIPLE \ -DLLVM_BUILD_32_BITS=OFF \ -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_ENABLE_THREADS=ON \ + -DLLVM_ENABLE_THREADS=$threading \ -DLLVM_DEFAULT_TARGET_TRIPLE=$TRIPLE \ -DLIBCXX_ENABLE_SHARED=OFF \ - -DLIBCXX_ENABLE_THREADS=ON \ + -DLIBCXX_ENABLE_THREADS=$threading \ -DLIBCXX_TARGET_TRIPLE=$TRIPLE \ -DLIBCXX_BUILD_32_BITS=OFF \ -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \ - -DLIBCXX_USE_COMPILER_RT=ON \ -DLIBCXX_CXX_ABI=libcxxabi \ -DLIBCXXABI_TARGET_TRIPLE=$TRIPLE \ - -DLIBCXXABI_ENABLE_THREADS=ON \ - -DLIBCXXABI_HAS_PTHREAD_API=ON \ + -DLIBCXXABI_ENABLE_THREADS=$threading \ + -DLIBCXXABI_HAS_PTHREAD_API=$threading \ -DLIBCXXABI_USE_LLVM_UNWINDER=ON \ -DLIBUNWIND_TARGET_TRIPLE=$TRIPLE \ - -DLIBUNWIND_TARGET_TRIPLE=$TRIPLE \ - -DLIBUNWIND_ENABLE_SHARED=OFF \ - -DCOMPILER_RT_BUILD_SANITIZERS=OFF - + -DLIBUNWIND_ENABLE_SHARED=OFF # MAKE +ninja libunwind.a ninja libc++abi.a - ninja libc++.a popd diff --git a/etc/create_binary_bundle.sh b/etc/create_binary_bundle.sh index 035900aefc..af28bbafc0 100755 --- a/etc/create_binary_bundle.sh +++ b/etc/create_binary_bundle.sh @@ -16,13 +16,13 @@ export num_jobs=${num_jobs:--j} # Specify number of build jobs # Version numbers export binutils_version=${binutils_version:-2.28} #ftp://ftp.gnu.org/gnu/binutils export musl_version=${musl_version:-v1.1.18} -export clang_version=${clang_version:-3.9} # http://releases.llvm.org/ -#export LLVM_TAG=${LLVM_TAG:-RELEASE_391/final} # http://llvm.org/svn/llvm-project/llvm/tags -export llvm_branch=${llvm_branch:-release_39} +export clang_version=${clang_version:-4.0} # http://releases.llvm.org/ +export llvm_branch=${llvm_branch:-release_40} # Options to skip steps [ ! -v do_binutils ] && do_binutils=1 [ ! -v do_musl ] && do_musl=1 +[ ! -v do_libunwind ] && do_libunwind=1; [ ! -v do_includeos ] && do_includeos=1 [ ! -v do_llvm ] && do_llvm=1 [ ! -v do_bridge ] && do_bridge=1 @@ -117,11 +117,10 @@ function do_build { $INCLUDEOS_SRC/etc/build_llvm.sh fi + # # Create the actual bundle # - - BUNDLE_DIR=$BUNDLE_PATH/$DIR_NAME BUNDLE_LOC=${BUNDLE_DIR}/$ARCH @@ -129,31 +128,47 @@ function do_build { musl=$TEMP_INSTALL_DIR/$TARGET/lib llvm=$BUILD_DIR/build_llvm - - libcpp=$llvm/lib/libc++.a - libcppabi=$llvm/lib/libc++abi.a libunwind=$llvm/lib/libunwind.a + libcppabi=$llvm/lib/libc++abi.a # Includes include_musl=$TEMP_INSTALL_DIR/$TARGET/include include_libcxx=$llvm/include/c++/v1 + include_libunwind=$BUILD_DIR/llvm/projects/libunwind/include/ # Make directory-tree mkdir -p $BUNDLE_LOC mkdir -p $BUNDLE_LOC/musl mkdir -p $BUNDLE_LOC/libcxx - + mkdir -p $BUNDLE_LOC/libunwind + mkdir -p $BUNDLE_LOC/libunwind/include # Copy binaries cp $libcpp $BUNDLE_LOC/libcxx/ cp $libcppabi $BUNDLE_LOC/libcxx/ - cp $libunwind $BUNDLE_LOC/libcxx/ + cp $libunwind $BUNDLE_LOC/libunwind/ cp -r $musl $BUNDLE_LOC/musl/ # Copy includes cp -r $include_musl $BUNDLE_LOC/musl/ cp -r $include_libcxx $BUNDLE_LOC/libcxx/include + cp $include_libunwind/libunwind.h $BUNDLE_LOC/libunwind/include + cp $include_libunwind/__libunwind_config.h $BUNDLE_LOC/libunwind/include + + if [ $ARCH == "i686" ] + then + crtbegin=`$CC -m32 --print-file-name crtbegin.o` + crtend=`$CC -m32 --print-file-name crtend.o` + else + crtbegin=`$CC --print-file-name crtbegin.o` + crtend=`$CC --print-file-name crtend.o` + fi + + mkdir -p $BUNDLE_LOC/crt + cp $crtbegin $BUNDLE_LOC/crt/ + cp $crtend $BUNDLE_LOC/crt/ + } for B_ARCH in $BUNDLE_ARCHES From 731370ec9a42ed05d520b858af82a138c517c7a2 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 18:31:02 +0100 Subject: [PATCH 030/723] gdb: add stepping mode --- etc/debug/service.gdb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/etc/debug/service.gdb b/etc/debug/service.gdb index 05c7e8973a..7566ca4802 100644 --- a/etc/debug/service.gdb +++ b/etc/debug/service.gdb @@ -1,9 +1,10 @@ -set architecture i386:x86-64 - +break OS::start break Service::start break main +set step-mode on set non-stop off target remote localhost:1234 -set $eax=1 -continue + +# To use the GDB_ENTRY macro, uncomment: +# set $eax=1 From ca39a28cfe50c690491dece66a2df07c42450737 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 19:25:00 +0100 Subject: [PATCH 031/723] musl: add compatibility patch --- etc/build_musl.sh | 10 +++++++-- etc/musl/musl.patch | 52 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) create mode 100644 etc/musl/musl.patch diff --git a/etc/build_musl.sh b/etc/build_musl.sh index 6d5e4dd4d6..1e0dd8236b 100755 --- a/etc/build_musl.sh +++ b/etc/build_musl.sh @@ -15,10 +15,13 @@ pushd musl # Replace syscall API cp $INCLUDEOS_SRC/api/syscalls.h src/internal/includeos_syscalls.h cp $INCLUDEOS_SRC/etc/musl/syscall.h src/internal/ - rm -f arch/x86_64/syscall_arch.h rm -f arch/i386/syscall_arch.h +# Compatibility patch +git apply $INCLUDEOS_SRC/etc/musl/musl.patch || true + + # ln -s $INCLUDEOS_SRC/api/arch/syscalls.h arch/x86_64/syscall_arch.h # ln -s $INCLUDEOS_SRC/api/arch/syscalls.h arch/i386/syscall_arch.h @@ -27,6 +30,9 @@ git checkout $musl_version make distclean || true export CFLAGS="$CFLAGS -target $ARCH-pc-linux-elf" -./configure --prefix=$TEMP_INSTALL_DIR/$TARGET --disable-shared --enable-optimize=* --target=$TARGET +./configure --prefix=$TEMP_INSTALL_DIR/$TARGET --disable-shared --enable-debug --target=$TARGET #--enable-optimize=* make $num_jobs make install + + +trap - EXIT diff --git a/etc/musl/musl.patch b/etc/musl/musl.patch new file mode 100644 index 0000000000..726f3bc9f1 --- /dev/null +++ b/etc/musl/musl.patch @@ -0,0 +1,52 @@ +diff --git a/arch/i386/atomic_arch.h b/arch/i386/atomic_arch.h +index 7d2a48a..0d9fc0f 100644 +--- a/arch/i386/atomic_arch.h ++++ b/arch/i386/atomic_arch.h +@@ -80,7 +80,7 @@ static inline void a_spin() + #define a_crash a_crash + static inline void a_crash() + { +- __asm__ __volatile__( "hlt" : : : "memory" ); ++ __asm__ __volatile__( "ud2" : : : "memory" ); + } + + #define a_ctz_64 a_ctz_64 +diff --git a/arch/x86_64/atomic_arch.h b/arch/x86_64/atomic_arch.h +index da4e203..08beb81 100644 +--- a/arch/x86_64/atomic_arch.h ++++ b/arch/x86_64/atomic_arch.h +@@ -105,7 +105,7 @@ static inline void a_spin() + #define a_crash a_crash + static inline void a_crash() + { +- __asm__ __volatile__( "hlt" : : : "memory" ); ++ __asm__ __volatile__( "ud2" : : : "memory" ); + } + + #define a_ctz_64 a_ctz_64 +diff --git a/src/thread/pthread_cancel.c b/src/thread/pthread_cancel.c +index 3d22922..a560b4f 100644 +--- a/src/thread/pthread_cancel.c ++++ b/src/thread/pthread_cancel.c +@@ -30,7 +30,7 @@ long __syscall_cp_c(syscall_arg_t nr, + + if ((st=(self=__pthread_self())->canceldisable) + && (st==PTHREAD_CANCEL_DISABLE || nr==SYS_close)) +- return __syscall(nr, u, v, w, x, y, z); ++ return syscall_n(nr, u, v, w, x, y, z); + + r = __syscall_cp_asm(&self->cancel, nr, u, v, w, x, y, z); + if (r==-EINTR && nr!=SYS_close && self->cancel && +diff --git a/src/unistd/setxid.c b/src/unistd/setxid.c +index 0239f8a..75c4165 100644 +--- a/src/unistd/setxid.c ++++ b/src/unistd/setxid.c +@@ -13,7 +13,7 @@ static void do_setxid(void *p) + { + struct ctx *c = p; + if (c->err>0) return; +- int ret = -__syscall(c->nr, c->id, c->eid, c->sid); ++ int ret = -syscall_n(c->nr, c->id, c->eid, c->sid); + if (ret && !c->err) { + /* If one thread fails to set ids after another has already + * succeeded, forcibly killing the process is the only safe From 15eec8a632943ec9659dae993a6b13ed80bc5eef Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 19:25:36 +0100 Subject: [PATCH 032/723] cmake: install chainloader from bundle --- cmake/cross_compiled_libraries.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/cross_compiled_libraries.txt b/cmake/cross_compiled_libraries.txt index 57edf57d85..03bd9b7813 100644 --- a/cmake/cross_compiled_libraries.txt +++ b/cmake/cross_compiled_libraries.txt @@ -106,6 +106,9 @@ install(DIRECTORY ${MUSL_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/musl install(FILES ${CRTEND} ${CRTBEGIN} DESTINATION includeos/${ARCH}/lib) +set(CHAINLOAD_LOC ${CMAKE_CURRENT_BINARY_DIR}/precompiled/src/PrecompiledLibraries/chainloader) +install(FILES ${CHAINLOAD_LOC} DESTINATION includeos) + file(GLOB musl_libs ${MUSL_LIB_DIR}/*.a) file(GLOB musl_objs ${MUSL_LIB_DIR}/*.o) From 2fa53141d7eba5cb9c91a54f655f1ba4188c3d5b Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 19:33:55 +0100 Subject: [PATCH 033/723] x86: enable syscall instruction --- etc/musl/syscall.h | 5 +- src/arch/x86_64/CMakeLists.txt | 8 +- src/arch/x86_64/arch_start.asm | 33 ++++++-- src/arch/x86_64/fiber_asm.asm | 4 +- src/arch/x86_64/syscall_entry.asm | 135 ++++++++++++++++++++++++++++++ src/crt/c_abi.c | 18 +--- 6 files changed, 168 insertions(+), 35 deletions(-) create mode 100644 src/arch/x86_64/syscall_entry.asm diff --git a/etc/musl/syscall.h b/etc/musl/syscall.h index 420f4138ea..525232f018 100644 --- a/etc/musl/syscall.h +++ b/etc/musl/syscall.h @@ -2,7 +2,6 @@ #define _INTERNAL_SYSCALL_H #include -//#include "syscall_arch.h"y #include "includeos_syscalls.h" @@ -45,8 +44,8 @@ long __syscall_ret(unsigned long), __syscall(syscall_arg_t, ...), #define socketcall __socketcall #define socketcall_cp __socketcall_cp -#define __syscall_cp syscall -#define syscall_cp syscall +#define __syscall_cp syscall +#define syscall_cp syscall #define __socketcall(nm,a,b,c,d,e,f) socketcall_##nm \ ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f }) diff --git a/src/arch/x86_64/CMakeLists.txt b/src/arch/x86_64/CMakeLists.txt index 7f9732d16a..8aea0b1309 100644 --- a/src/arch/x86_64/CMakeLists.txt +++ b/src/arch/x86_64/CMakeLists.txt @@ -8,15 +8,13 @@ set(ARCH_OBJECTS exceptions.asm interrupts.asm fiber_asm.asm + syscall_entry.asm ist.cpp ) add_library(arch STATIC ${ARCH_OBJECTS}) -add_library(crti STATIC crti.asm) -add_library(crtn STATIC crtn.asm) - add_dependencies(arch PrecompiledLibraries) -set_target_properties(crti crtn arch PROPERTIES LINKER_LANGUAGE CXX) +set_target_properties(arch PROPERTIES LINKER_LANGUAGE CXX) -install(TARGETS crti crtn arch DESTINATION includeos/${ARCH}/lib) +install(TARGETS arch DESTINATION includeos/${ARCH}/lib) install(FILES linker.ld DESTINATION includeos/${ARCH}) diff --git a/src/arch/x86_64/arch_start.asm b/src/arch/x86_64/arch_start.asm index cec61dfd91..7e9b9946d3 100644 --- a/src/arch/x86_64/arch_start.asm +++ b/src/arch/x86_64/arch_start.asm @@ -20,11 +20,23 @@ extern kernel_start extern __multiboot_magic extern __multiboot_addr -%define PAGE_SIZE 0x1000 -%define P4_TAB 0x1000 -%define P3_TAB 0x2000 ;; - 0x5fff -%define P2_TAB 0x100000 -%define STACK_LOCATION 0x9ffff0 +%define PAGE_SIZE 0x1000 +%define P4_TAB 0x1000 +%define P3_TAB 0x2000 ;; - 0x5fff +%define P2_TAB 0x100000 +%define STACK_LOCATION 0x9ffff0 + +%define EFER.LME 0x100 +%define EFER.SCE 0x1 + +%define IA32_EFER 0xC0000080 +%define IA32_STAR 0xC0000081 +%define IA32_LSTAR 0xc0000082 +%define IA32_FMASK 0xc0000084 +%define IA32_FS_BASE 0xc0000100 +%define IA32_GS_BASE 0xc0000101 +%define IA32_KERNEL_GS_BASE 0xc0000102 + [BITS 32] __arch_start: @@ -77,9 +89,9 @@ __arch_start: mov cr4, eax ;; enable long mode - mov ecx, 0xC0000080 ; EFER MSR + mov ecx, IA32_EFER ; EFER MSR rdmsr - or eax, 1 << 8 ; Long Mode bit + or eax, EFER.LME | EFER.SCE ; Long M wrmsr ;; enable paging @@ -112,7 +124,12 @@ long_mode: ;; setup temporary smp table mov rax, sentinel_table mov rdx, 0 - mov rcx, 0xC0000100 ;; FS BASE + mov rcx, IA32_FS_BASE ;; FS BASE + wrmsr + + mov ecx, IA32_STAR + mov edx, 0x8 + mov eax, 0x0 wrmsr ;; geronimo! diff --git a/src/arch/x86_64/fiber_asm.asm b/src/arch/x86_64/fiber_asm.asm index 1792380e0c..5c2c812484 100644 --- a/src/arch/x86_64/fiber_asm.asm +++ b/src/arch/x86_64/fiber_asm.asm @@ -30,7 +30,7 @@ extern fiber_jumpstarter %define arg5 r8 %define arg6 r9 -;; Preserve caller-saved registers +;; Preserve callee-saved registers %macro PUSHEM 0 push rbp push rbx @@ -40,7 +40,7 @@ extern fiber_jumpstarter push r15 %endmacro -;; Restore caller-saved registers +;; Restore callee-saved registers %macro POPEM 0 pop r15 pop r14 diff --git a/src/arch/x86_64/syscall_entry.asm b/src/arch/x86_64/syscall_entry.asm new file mode 100644 index 0000000000..909110b768 --- /dev/null +++ b/src/arch/x86_64/syscall_entry.asm @@ -0,0 +1,135 @@ +;; This file is a part of the IncludeOS unikernel - www.includeos.org +;; +;; Copyright 2018 IncludeOS AS, Oslo, Norway +;; +;; Licensed under the Apache License, Version 2.0 (the "License"); +;; you may not use this file except in compliance with the License. +;; You may obtain a copy of the License at +;; +;; http:;;www.apache.org/licenses/LICENSE-2.0 +;; +;; Unless required by applicable law or agreed to in writing, software +;; distributed under the License is distributed on an "AS IS" BASIS, +;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +;; See the License for the specific language governing permissions and +;; limitations under the License. + +global __syscall_entry:function +global __test_syscall:function +extern syscall_entry +extern kprintf + +;; x86_64 / System V ABI calling convention +%define arg1 rdi +%define arg2 rsi +%define arg3 rdx +%define arg4 rcx +%define arg5 r8 +%define arg6 r9 + +;; Preserve caller-saved registers +%macro PUSHAQ 0 + push rax + push rcx + push rdx + push rdi + push rsi + push r8 + push r9 + push r10 + push r11 + ; Preserve extended state + ;fxsave [__xsave_storage_area] +%endmacro +%macro POPAQ 0 + ; Restore extended state + ;fxrstor [__xsave_storage_area] + + pop r11 + pop r10 + pop r9 + pop r8 + pop rsi + pop rdi + pop rdx + pop rcx + pop rax +%endmacro + +%define stack_size 32768 + +section .bss +temp_stack: + resb stack_size + +section .data: +temp_old_stack: + dq 0 +temp_rcx: + dq 0 + +section .text +__syscall_entry: + ;; mov [temp_rcx], rcx + ;; mov [temp_old_stack], rsp + ;; mov rsp, temp_stack + stack_size - 16 + + push rcx + ;; sub rsp, 8 + + ;; Convert back from syscall conventions + ;; Last param on stack movq 8(rsp),r9 + mov r9, r8 + mov r8, r10 + mov rcx, rdx + mov rdx, rsi + mov rsi, rdi + mov rdi, rax + + call syscall_entry + ;; add rsp, 8 + pop rcx + + ;; mov rsp, [temp_old_stack] + jmp QWORD rcx ;[temp_rcx] + + +__test_syscall: + mov [kparam.stack], rsp + mov arg1, kparam.rsp + mov arg2, [kparam.stack] + mov arg3, 0 + + push rsp + and rsp, ~15 + call kprintf + pop rsp + + mov rsi, rdi ; shift for syscall + mov edi, 0x1002 ;/* SET_FS register */ + mov eax, 158 ;/* set fs segment to */ + + + syscall ;/* arch_prctl(SET_FS, arg)*/ + + mov [kparam.stack], rsp + mov arg1, kparam.rsp + mov arg2, [kparam.stack] + mov arg3, 0 + + push rsp + and rsp, ~15 + call kprintf + pop rsp + ret + +;; Make thread local +kparam: + .rsp: + db `__test_syscall: Stack: 0x%lx\n`,0 + .fmt_rcx: + db `__test_syscall: rcx: 0x%lx\n`,0 + .stack: + dq 0 + .rcx: + dq 0 diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index b9e9d448be..a55079330a 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -98,25 +98,9 @@ static void crt_sanity_checks() assert(validate_heap_alignment); } + void _init_c_runtime() { - /// initialize newlib I/O - // _REENT_INIT_PTR(_REENT); - // Unix standard streams - - /* - stdin = _REENT->_stdin; // stdin == 1 - stdout = _REENT->_stdout; // stdout == 2 - stderr = _REENT->_stderr; // stderr == 3 - */ - - /// initialize exceptions before we can run constructors - extern char __eh_frame_start[]; - // Tell the stack unwinder where exception frames are located - extern void __register_frame(void*); - kprintf("Not registering frame... \n"); - //__register_frame(&__eh_frame_start); - //kprintf("Registering frame OK \n"); /// init ELF / backtrace functionality extern void _init_elf_parser(); From c38232d5903dafd73e71b75f053f724cae5a0d54 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 19:41:39 +0100 Subject: [PATCH 034/723] musl: more syscall stubs + more or less working brk --- src/musl/CMakeLists.txt | 4 +++- src/musl/_lseek.cpp | 7 ++++++ src/musl/brk.cpp | 35 ++++++++++++++++++--------- src/musl/close.cpp | 7 ++++-- src/musl/common.hpp | 4 ++-- src/musl/futex.cpp | 39 +++++++++++++++++++++++++++++-- src/musl/getpid.cpp | 7 ++++++ src/musl/ioctl.cpp | 23 ++++++++++++++++-- src/musl/lseek.cpp | 6 +++++ src/musl/mincore.cpp | 7 ++++++ src/musl/mknod.cpp | 13 +++++++++++ src/musl/mmap.cpp | 4 ++-- src/musl/{fstat.cpp => msync.cpp} | 4 ++-- src/musl/open.cpp | 14 +++++++---- src/musl/pipe.cpp | 13 +++++++++++ src/musl/poll.cpp | 2 ++ src/musl/read.cpp | 7 ++++++ src/musl/readv.cpp | 26 +++++++++++++++++++++ src/musl/set_tid_address.cpp | 23 +++++++++++++++--- src/musl/sigmask.cpp | 35 +++++++++++++++++++++++++++ src/musl/stat.cpp | 25 ++++++++++++++++++++ src/musl/sync.cpp | 13 +++++++++++ src/musl/syscall_n.cpp | 7 ++++++ src/musl/write.cpp | 5 +++- 24 files changed, 298 insertions(+), 32 deletions(-) create mode 100644 src/musl/_lseek.cpp create mode 100644 src/musl/getpid.cpp create mode 100644 src/musl/mincore.cpp create mode 100644 src/musl/mknod.cpp rename src/musl/{fstat.cpp => msync.cpp} (52%) create mode 100644 src/musl/pipe.cpp create mode 100644 src/musl/read.cpp create mode 100644 src/musl/readv.cpp create mode 100644 src/musl/sigmask.cpp create mode 100644 src/musl/stat.cpp create mode 100644 src/musl/sync.cpp create mode 100644 src/musl/syscall_n.cpp diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index 6602b237fc..bfa96b4d9f 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -4,8 +4,10 @@ set(MUSL_OBJECTS write.cpp brk.cpp madvise.cpp mmap.cpp mremap.cpp munmap.cpp lseek.cpp sched_getaffinity.cpp sysinfo.cpp prlimit64.cpp getrlimit.cpp sched_yield.cpp set_robust_list.cpp - nanosleep.cpp open.cpp fstat.cpp clock_gettime.cpp gettimeofday.cpp + nanosleep.cpp open.cpp clock_gettime.cpp gettimeofday.cpp poll.cpp mmap2.cpp exit_group.cpp exit.cpp close.cpp set_tid_address.cpp + pipe.cpp read.cpp readv.cpp getpid.cpp mknod.cpp stat.cpp sync.cpp msync.cpp + mincore.cpp syscall_n.cpp sigmask.cpp ) add_library(musl_syscalls STATIC ${MUSL_OBJECTS}) diff --git a/src/musl/_lseek.cpp b/src/musl/_lseek.cpp new file mode 100644 index 0000000000..362b3326e4 --- /dev/null +++ b/src/musl/_lseek.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS__lseek() { + STUB("_lseek"); + return 0; +} diff --git a/src/musl/brk.cpp b/src/musl/brk.cpp index 38aa54a604..761223fec5 100644 --- a/src/musl/brk.cpp +++ b/src/musl/brk.cpp @@ -1,24 +1,37 @@ #include "common.hpp" -#include +#include #include #include uintptr_t heap_begin; uintptr_t heap_end; +uintptr_t max_brk; + +#warning stub extern "C" -int syscall_SYS_brk(void* end_data_segment) +uintptr_t syscall_SYS_brk(void* addr) { - uintptr_t actual = heap_begin + (uintptr_t)end_data_segment; - int res = actual <= OS::heap_max() ? 0 : -1; + STRACE("syscall brk. Heap begin=0x%lx, OS::heap_max=0x%lx, addr=%p" + "\n\t requests %i bytes \n", + heap_begin, OS::heap_max(), addr, (uintptr_t)addr - heap_end); - if(res == 0) - { - heap_end = actual; - } + // Keep track of total used heap + if (max_brk == 0) + max_brk = heap_end; - STRACE("syscall brk(end_data_seg=%p) = %d\n", - end_data_segment, res); + if (addr == nullptr + or (uintptr_t)addr > OS::heap_max() + or (uintptr_t)addr < heap_begin) + return heap_end; - return res; + heap_end = (uintptr_t)addr; + + if (heap_end > max_brk) { + kprintf("\t Initializing %i b\n", heap_end - max_brk); + memset((void*)max_brk, 0, heap_end - max_brk); + max_brk = heap_end; + } + kprintf("\t Done, returning 0x%lx\n", heap_end); + return heap_end; } diff --git a/src/musl/close.cpp b/src/musl/close.cpp index 3275273470..40466a61f7 100644 --- a/src/musl/close.cpp +++ b/src/musl/close.cpp @@ -1,7 +1,10 @@ #include "common.hpp" extern "C" -long syscall_SYS_close() { - STUB("close"); +int syscall_SYS_close(int fd) { + STRACE("close fd=%i\n", fd); + + if (!fd) return -1; + return 0; } diff --git a/src/musl/common.hpp b/src/musl/common.hpp index e58f7f443d..3c3a78b7f8 100644 --- a/src/musl/common.hpp +++ b/src/musl/common.hpp @@ -2,6 +2,6 @@ #include -#define STUB(X) kprintf("syscall %s called\n", X) +#define STUB(X) kprintf(" stubbed syscall %s called\n", X) -#define STRACE(X, ...) kprintf(X, ##__VA_ARGS__) +#define STRACE(X, ...) kprintf(" " X, ##__VA_ARGS__) diff --git a/src/musl/futex.cpp b/src/musl/futex.cpp index cbef165067..e0aa66685c 100644 --- a/src/musl/futex.cpp +++ b/src/musl/futex.cpp @@ -1,7 +1,42 @@ #include "common.hpp" +#include + +#define FUTEX_WAIT 0 +#define FUTEX_WAKE 1 +#define FUTEX_FD 2 +#define FUTEX_REQUEUE 3 +#define FUTEX_CMP_REQUEUE 4 +#define FUTEX_WAKE_OP 5 +#define FUTEX_LOCK_PI 6 +#define FUTEX_UNLOCK_PI 7 +#define FUTEX_TRYLOCK_PI 8 +#define FUTEX_WAIT_BITSET 9 + +#define FUTEX_PRIVATE 128 + +#define FUTEX_CLOCK_REALTIME 256 + +extern "C" void panic(char*); +extern void print_backtrace(); extern "C" -long syscall_SYS_futex() { - STUB("Futex"); +int syscall_SYS_futex(int *uaddr, int futex_op, int val, + const struct timespec *timeout, int val3) +{ + STRACE("syscall futex: uaddr=%p futex_op=0x%x val=%i timeout=%p, val3: %i\n", + uaddr, futex_op, val, timeout, val3); + + //print_backtrace(); + if (*uaddr != val){ + return EAGAIN; + } else { + *uaddr = 0; + } + + if (timeout == nullptr){ + kprintf("No timeout\n"); + } + + kprintf("Futex ok\n"); return 0; } diff --git a/src/musl/getpid.cpp b/src/musl/getpid.cpp new file mode 100644 index 0000000000..25469c1571 --- /dev/null +++ b/src/musl/getpid.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_getpid() { + STRACE("getpid"); + return 0; +} diff --git a/src/musl/ioctl.cpp b/src/musl/ioctl.cpp index 67cf4d1843..4f5b0de890 100644 --- a/src/musl/ioctl.cpp +++ b/src/musl/ioctl.cpp @@ -1,7 +1,26 @@ #include "common.hpp" +#include + +#warning stub extern "C" -long syscall_SYS_ioctl() { - STUB("ioctl"); +int syscall_SYS_ioctl(int fd, int req, void* arg) { + STRACE("syscall ioctl: fd=%i, req=0x%x\n", + fd, req); + + if (req == TIOCGWINSZ) { + kprintf("\t* Wants to get winsize\n"); + winsize* ws = (winsize*)arg; + ws->ws_row = 25; + ws->ws_col = 80; + return 0; + } + + if (req == TIOCSWINSZ) { + winsize* ws = (winsize*)arg; + kprintf("\t* Wants to set winsize to %i x %i\n", + ws->ws_row, ws->ws_col); + } + return 0; } diff --git a/src/musl/lseek.cpp b/src/musl/lseek.cpp index ff3bcbec01..16907da960 100644 --- a/src/musl/lseek.cpp +++ b/src/musl/lseek.cpp @@ -5,3 +5,9 @@ long syscall_SYS_lseek() { STUB("lseek"); return 0; } + +extern "C" +long syscall_SYS__llseek() { + STUB("_lseek"); + return 0; +} diff --git a/src/musl/mincore.cpp b/src/musl/mincore.cpp new file mode 100644 index 0000000000..d0cd2d7d2e --- /dev/null +++ b/src/musl/mincore.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_mincore() { + STUB("mincore"); + return 0; +} diff --git a/src/musl/mknod.cpp b/src/musl/mknod.cpp new file mode 100644 index 0000000000..1814890db7 --- /dev/null +++ b/src/musl/mknod.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" + +extern "C" { +long syscall_SYS_mknod() { + STUB("mknod"); + return 0; +} + +long syscall_SYS_mknodat() { + STUB("mknod"); + return 0; +} +} diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index db2aeec27e..511b92aefd 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -9,8 +9,8 @@ void* syscall_SYS_mmap(void *addr, size_t length, int prot, int flags, { uintptr_t res = heap_begin + current_pos; current_pos += length; - STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d\n", - addr, length, prot, flags, fd, offset); + STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d res=%p\n", + addr, length, prot, flags, fd, offset, res); return (void*)res; } diff --git a/src/musl/fstat.cpp b/src/musl/msync.cpp similarity index 52% rename from src/musl/fstat.cpp rename to src/musl/msync.cpp index c8903de30b..b69db69ceb 100644 --- a/src/musl/fstat.cpp +++ b/src/musl/msync.cpp @@ -1,7 +1,7 @@ #include "common.hpp" extern "C" -long syscall_SYS_fstat() { - STUB("fstat"); +long syscall_SYS_msync() { + STUB("msync"); return 0; } diff --git a/src/musl/open.cpp b/src/musl/open.cpp index 9eb3d00aaf..fa1c8a16e4 100644 --- a/src/musl/open.cpp +++ b/src/musl/open.cpp @@ -1,7 +1,13 @@ #include "common.hpp" +#include -extern "C" -long syscall_SYS_open() { - STUB("open"); - return 0; +extern "C" { +int syscall_SYS_open(const char *pathname, int flags, mode_t mode = 0) { + STRACE("open pathname=%s, flags=0x%x, mode=0x%x", + pathname, flags, mode); } + +int syscall_SYS_creat(const char *pathname, mode_t mode) { + STUB("creat"); +} +} // extern "C" diff --git a/src/musl/pipe.cpp b/src/musl/pipe.cpp new file mode 100644 index 0000000000..f0668a02fa --- /dev/null +++ b/src/musl/pipe.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" + +extern "C" { +long syscall_SYS_pipe() { + STUB("pipe"); + return 0; +} + +long syscall_SYS_pipe2() { + STUB("pipe2"); + return 0; +} +} // extern "C" diff --git a/src/musl/poll.cpp b/src/musl/poll.cpp index eb68e6219a..56901ad0dd 100644 --- a/src/musl/poll.cpp +++ b/src/musl/poll.cpp @@ -1,6 +1,8 @@ #include "common.hpp" #include +#warning stub + extern "C" long syscall_SYS_poll(struct pollfd *fds, nfds_t nfds, int timeout) { diff --git a/src/musl/read.cpp b/src/musl/read.cpp new file mode 100644 index 0000000000..dafb5fbf9b --- /dev/null +++ b/src/musl/read.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_read() { + STUB("read"); + return 0; +} diff --git a/src/musl/readv.cpp b/src/musl/readv.cpp new file mode 100644 index 0000000000..afbdbc2cb2 --- /dev/null +++ b/src/musl/readv.cpp @@ -0,0 +1,26 @@ +#include "common.hpp" +#include +#include + +extern "C" +ssize_t syscall_SYS_readv(int fd, const struct iovec *iov, int iovcnt) + +{ + /* + if(fd <= 2) + { + ssize_t res = 0; + for(int i = 0; i < iovcnt; i++) + { + auto* text = (const char*)iov[i].iov_base; + auto len = iov[i].iov_len; + kprintf("%.*s", len, text); + res += len; + } + return res; + }*/ + STRACE("syscall readv: fd=%d iov=%p iovcount=%d [base=%p sz=%d]\n", + fd, iov, iovcnt, iov->iov_base, iov->iov_len); + errno = ENOSYS; + return -1; +} diff --git a/src/musl/set_tid_address.cpp b/src/musl/set_tid_address.cpp index b07864ad96..b9b51f1469 100644 --- a/src/musl/set_tid_address.cpp +++ b/src/musl/set_tid_address.cpp @@ -1,7 +1,24 @@ #include "common.hpp" +#warning stub + +#ifdef INCLUDEOS_SINGLE_THREADED +struct { + int tid = 1; + int* set_child_tid = nullptr; + int* clear_child_tid = nullptr; +} __main_thread__; +#else +#warning syscall set_tid_address is not yet thread safe +#endif + extern "C" -long syscall_SYS_set_tid_address() { - STUB("set_tid_address"); - return 0; +long syscall_SYS_set_tid_address(int* tidptr) { + STRACE("set_tid_address, %p\n", tidptr); + if (tidptr) { + kprintf("\t*tidptr: 0x%x\n", *tidptr); + } + __main_thread__.clear_child_tid = tidptr; + + return __main_thread__.tid; } diff --git a/src/musl/sigmask.cpp b/src/musl/sigmask.cpp new file mode 100644 index 0000000000..fb32a635ee --- /dev/null +++ b/src/musl/sigmask.cpp @@ -0,0 +1,35 @@ +#include "common.hpp" +#include + +extern "C" { +long syscall_SYS_sigmask() { + STUB("sigmask"); + return 0; +} + +long syscall_SYS_rt_sigprocmask(int how, const sigset_t *set, sigset_t *oldset) { + +char* howstr = nullptr; + +switch(how) { + + case (SIG_BLOCK): +howstr="BLOCK"; +break; + + case (SIG_UNBLOCK): +howstr="UNBLOCK"; +break; + + case (SIG_SETMASK): +howstr="SETMASK"; +break; + +} + +STRACE("rt_sigprocmask how=%i (%s), set=%p, oldset=%p\n", + how, howstr, set, oldset); + +return 0; +} +} // extern "C" diff --git a/src/musl/stat.cpp b/src/musl/stat.cpp new file mode 100644 index 0000000000..bb054b07ba --- /dev/null +++ b/src/musl/stat.cpp @@ -0,0 +1,25 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_stat() { + STUB("stat"); + return 0; +} + +extern "C" +long syscall_SYS_lstat() { + STUB("lstat"); + return 0; +} + +extern "C" +long syscall_SYS_fstat() { + STUB("fstat"); + return 0; +} + +extern "C" +long syscall_SYS_fstatat() { + STUB("fstatat"); + return 0; +} diff --git a/src/musl/sync.cpp b/src/musl/sync.cpp new file mode 100644 index 0000000000..a51f892e75 --- /dev/null +++ b/src/musl/sync.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" + +extern "C" { +long syscall_SYS_sync() { + STUB("sync"); + return 0; +} + +long syscall_SYS_syncfs() { + STUB("syncfs"); + return 0; +} +} diff --git a/src/musl/syscall_n.cpp b/src/musl/syscall_n.cpp new file mode 100644 index 0000000000..9ff6cd2ec3 --- /dev/null +++ b/src/musl/syscall_n.cpp @@ -0,0 +1,7 @@ +#include "common.hpp" + +extern "C" +long syscall_n(long i) { + STRACE("syscall_n, n=%i\n", i); + return 0; +} diff --git a/src/musl/write.cpp b/src/musl/write.cpp index 8b69595c0e..8fae36001e 100644 --- a/src/musl/write.cpp +++ b/src/musl/write.cpp @@ -1,7 +1,10 @@ #include "common.hpp" +extern "C" void __serial_print(const char*, size_t); extern "C" -long syscall_SYS_write() { +long syscall_SYS_write(int fd, char* str, size_t len) { STUB("write"); + + __serial_print(str, len); return 0; } From a65bdd9ca41548285a2ddb7c2e425e488f123f02 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 19:45:08 +0100 Subject: [PATCH 035/723] kprint: make proper function --- src/include/kprint | 13 ++----------- src/kernel/syscalls.cpp | 16 ---------------- src/platform/x86_pc/serial1.cpp | 12 +++++++++++- 3 files changed, 13 insertions(+), 28 deletions(-) diff --git a/src/include/kprint b/src/include/kprint index 31869e1c9f..f00ac745b4 100644 --- a/src/include/kprint +++ b/src/include/kprint @@ -39,17 +39,8 @@ extern void __serial_print(const char* str, size_t len); * The earliest possible print function (requires no heap, global ctors etc.) **/ __attribute__ ((format (printf, 1, 2))) -inline void kprintf(const char* format, ...) -{ - char buf[8192]; - va_list aptr; - va_start(aptr, format); - vsnprintf(buf, sizeof(buf), format, aptr); - __serial_print1(buf); - va_end(aptr); -} - -#define kprint(cstr) __serial_print1(cstr) +extern "C" void kprintf(const char* format, ...); +extern void kprint(const char*); #ifdef __cplusplus } diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index 7003e26536..b8b920967e 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -53,22 +53,6 @@ void abort() { kprintf("Abooooooring!\n"); panic("abort() called"); } -extern "C" -void abort_message(const char* fmt, ...) -{ - kprintf("Aboooooort message %s \n", fmt); - char buffer[1024]; - va_list list; - va_start(list, fmt); - vsnprintf(buffer, sizeof(buffer), fmt, list); - va_end(list); -#ifdef ARCH_x86_64 - asm("movq %0, %%rdi" : : "r"(buffer)); - asm("jmp panic_begin"); -#else - panic(buffer); -#endif -} void _exit(int status) { kprintf("%s",std::string(LINEWIDTH, '=').c_str()); diff --git a/src/platform/x86_pc/serial1.cpp b/src/platform/x86_pc/serial1.cpp index 101b62b116..c51b169457 100644 --- a/src/platform/x86_pc/serial1.cpp +++ b/src/platform/x86_pc/serial1.cpp @@ -1,5 +1,5 @@ #include - +#include static const uint16_t port = 0x3F8; // Serial 1 extern "C" @@ -37,3 +37,13 @@ extern "C" void kprint(const char* c){ __serial_print1(c); } + +extern "C" void kprintf(const char* format, ...) +{ + char buf[8192]; + va_list aptr; + va_start(aptr, format); + vsnprintf(buf, sizeof(buf), format, aptr); + __serial_print1(buf); + va_end(aptr); +} From fa0ae6e683b2f9945e821d38c2b3ca15fe219729 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 19:51:28 +0100 Subject: [PATCH 036/723] musl: using musl's own initialization functions --- src/platform/x86_pc/cpu.hpp | 10 ++ src/platform/x86_pc/gdt.hpp | 7 +- src/platform/x86_pc/kernel_start.cpp | 176 +++++++++++++++++---- src/platform/x86_pc/musl_init.cpp | 155 ------------------ src/platform/x86_pc/platform.cpp | 5 +- test/kernel/integration/tls/CMakeLists.txt | 3 + 6 files changed, 165 insertions(+), 191 deletions(-) delete mode 100644 src/platform/x86_pc/musl_init.cpp diff --git a/src/platform/x86_pc/cpu.hpp b/src/platform/x86_pc/cpu.hpp index 7d174db6b3..0e3cbfcf54 100644 --- a/src/platform/x86_pc/cpu.hpp +++ b/src/platform/x86_pc/cpu.hpp @@ -21,9 +21,19 @@ #include #include +#include + +#define IA32_EFER 0xC0000080 +#define IA32_STAR 0xC0000081 +#define IA32_LSTAR 0xc0000082 +#define IA32_FMASK 0xc0000084 +#define IA32_FS_BASE 0xC0000100 +#define IA32_GS_BASE 0xC0000101 +#define IA32_KERNEL_GS_BASE 0xC0000102 namespace x86 { + class CPU { public: diff --git a/src/platform/x86_pc/gdt.hpp b/src/platform/x86_pc/gdt.hpp index b60e9bed23..4a07606f4e 100644 --- a/src/platform/x86_pc/gdt.hpp +++ b/src/platform/x86_pc/gdt.hpp @@ -49,15 +49,12 @@ struct GDT static void reload_gdt(GDT& base) noexcept; #if defined(ARCH_x86_64) -#define MSR_FS_BASE 0xC0000100 -#define MSR_GS_BASE 0xC0000101 -#define MSR_GS_SWAP 0xC0000102 static inline void set_fs(void* entry) noexcept { - CPU::write_msr(MSR_FS_BASE, (uintptr_t) entry); + CPU::write_msr(IA32_FS_BASE, (uintptr_t) entry); } static inline void set_gs(void* entry) noexcept { - CPU::write_msr(MSR_GS_BASE, (uintptr_t) entry); + CPU::write_msr(IA32_GS_BASE, (uintptr_t) entry); } #elif defined(ARCH_i686) static inline void set_fs(uint16_t entry) noexcept { diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 6abc1bd068..3fc8f6960b 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -1,7 +1,6 @@ // This file is a part of the IncludeOS unikernel - www.includeos.org // -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud +// Copyright 2015-2018 IncludeOS AS, Oslo, Norway // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -16,21 +15,33 @@ // limitations under the License. #include +#include #include #include #include -#include "musl_init.cpp" +#include +#include +#include "cpu.hpp" +#include "gdt.hpp" extern "C" { void __init_serial1(); void __init_sanity_checks(); void kernel_sanity_checks(); + void _init_bss(); + void __test_syscall(void* p); uintptr_t _multiboot_free_begin(uintptr_t boot_addr); - //uintptr_t _move_symbols(uintptr_t loc); + uintptr_t _move_symbols(uintptr_t loc); } extern uintptr_t heap_begin; extern uintptr_t heap_end; +extern char _ELF_START_; +extern char _ELF_END_; +extern char _INIT_START_; +extern char _FINI_START_; + +thread_local int __tl1__ = 42; void _init_heap(uintptr_t free_mem_begin) { @@ -40,12 +51,28 @@ void _init_heap(uintptr_t free_mem_begin) heap_begin = free_mem_begin + HEAP_ALIGNMENT; heap_begin = ((uintptr_t)heap_begin & ~HEAP_ALIGNMENT); heap_end = heap_begin; + kprintf("* Heap initialized. Begin: 0x%lx, end 0x%lx\n", + heap_begin, heap_end); } +void _init_bss() +{ + /// Initialize .bss section + extern char _BSS_START_, _BSS_END_; + __builtin_memset(&_BSS_START_, 0, &_BSS_END_ - &_BSS_START_); +} + +uintptr_t __grub_magic = 0xc001; +uintptr_t __grub_addr = 0x7001; + int kernel_main(int, char * *, char * *) { - kprintf("kernel_main() !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!\n"); + kprintf(" libc initialization complete \n"); + Expects(__tl1__ == 42); + Elf_binary elf{{(char*)&_ELF_START_, &_ELF_END_ - &_ELF_START_}}; + Expects(elf.is_ELF() && "ELF header intact"); + // Initialize early OS, platform and devices - OS::start(0u,0u); + OS::start(__grub_magic,__grub_addr); // Initialize common subsystems and call Service::start OS::post_start(); @@ -55,6 +82,7 @@ int kernel_main(int, char * *, char * *) { // Starting event loop from here allows us to profile OS::start OS::event_loop(); + return 0; } void __init_tls(size_t* p) @@ -62,19 +90,62 @@ void __init_tls(size_t* p) kprintf("init_tls(%p)\n", p); } +// Musl entry extern "C" -int __libc_start_main(int *(main) (int, char * *, char * *), - int argc, char * * ubp_av, - void (*init) (void), - void (*fini) (void), - void (*rtld_fini) (void), - void (* stack_end)); +int __libc_start_main(int (*main)(int,char **,char **), int argc, char **argv); extern "C" void _init(); extern "C" void _fini(); #include #include + +#define ARCH_SET_GS 0x1001 +#define ARCH_SET_FS 0x1002 +#define ARCH_GET_FS 0x1003 +#define ARCH_GET_GS 0x1004 + +int __arch_prctl(int code, uintptr_t ptr){ + + switch(code){ + case ARCH_SET_GS: + kprintf(" set_gs to %#lx\n", ptr); + if (!ptr) return -1; + x86::GDT::set_gs((void*)ptr); + break; + case ARCH_SET_FS: + kprintf(" set_fs to %#lx\n", ptr); + if (!ptr) return -1; + x86::GDT::set_fs((void*)ptr); + break; + case ARCH_GET_GS: + panic(" get gs \n"); + break; + case ARCH_GET_FS: + panic(" get gs \n"); + break; + } + return 0; +} + + +extern "C" +uintptr_t syscall_entry(uint64_t n, uint64_t a1, uint64_t a2, uint64_t a3, + uint64_t a4, uint64_t a5) +{ + kprintf(" no %lu (a1=%#lx a2=%#lx a3=%#lx a4=%#lx a5=%#lx) \n", + n, a1, a2, a3, a4, a5); + + switch(n) { + case 158: // arch_prctl + __arch_prctl(a1, a2); + break; + } + return 0; +} + +extern "C" uintptr_t __syscall_entry(); + extern "C" __attribute__((no_sanitize("all"))) void kernel_start(uintptr_t magic, uintptr_t addr) @@ -82,43 +153,76 @@ void kernel_start(uintptr_t magic, uintptr_t addr) // Initialize default serial port __init_serial1(); + __grub_magic = magic; + __grub_addr = addr; + + // TODO: remove extra verbosity after musl port stabilizes + kprintf("\n////////////////// IncludeOS kernel start ////////////////// \n", + magic, addr); + kprintf("* Booted with magic 0x%lx, grub @ 0x%lx \n* Init sanity\n", magic, addr); // generate checksums of read-only areas etc. __init_sanity_checks(); + kprintf("* Grub magic: 0x%lx, grub info @ 0x%lx\n", + __grub_magic, __grub_addr); + // Determine where free memory starts extern char _end; uintptr_t free_mem_begin = reinterpret_cast(&_end); if (magic == MULTIBOOT_BOOTLOADER_MAGIC) { free_mem_begin = _multiboot_free_begin(addr); + kprintf("* Free mem begin: %p \n", free_mem_begin); } // Preserve symbols from the ELF binary //free_mem_begin += _move_symbols(free_mem_begin); - // TODO: set heap begin + kprintf("* Grub magic: 0x%lx, grub info @ 0x%lx\n", + __grub_magic, __grub_addr); + + kprintf("* Init .bss\n"); + _init_bss(); + + + kprintf("* Init heap\n"); _init_heap(free_mem_begin); - extern char _INIT_START_; - extern char _FINI_START_; - void* init_location = &_INIT_START_; - void* fini_location = &_FINI_START_; - void* rtld_fini = nullptr; - void* stack_end = (void*) 0x10000; + kprintf("* Thread local1: %i\n", __tl1__); + + + kprintf("* Elf start: 0x%lx\n", &_ELF_START_); + auto* ehdr = (Elf64_Ehdr*)&_ELF_START_; + auto* phdr = (Elf64_Phdr*)((char*)ehdr + ehdr->e_phoff); + Elf_binary elf{{(char*)&_ELF_START_, &_ELF_END_ - &_ELF_START_}}; + Expects(elf.is_ELF()); + size_t size = &_ELF_END_ - &_ELF_START_; + Expects(phdr[0].p_type == PT_LOAD); + + printf("* Elf ident: %s, program headers:\n", ehdr->e_ident, ehdr); + kprintf("\tElf size: %i \n", size); + for (int i = 0; i < ehdr->e_phnum; i++) + { + kprintf("\tPhdr %i @ %p, va_addr: 0x%lx \n", i, &phdr[i], phdr[i].p_vaddr); + } auxv_t aux[38]; + kprintf("* Initializing aux-vector @ %p\n", aux); + int i = 0; aux[i++].set_long(AT_PAGESZ, 4096); aux[i++].set_long(AT_CLKTCK, 100); - aux[i++].set_long(AT_PHENT, 32); - aux[i++].set_ptr(AT_PHDR, 0x0); - aux[i++].set_long(AT_PHNUM, 0); + // ELF related + aux[i++].set_long(AT_PHENT, ehdr->e_phentsize); + aux[i++].set_ptr(AT_PHDR, ((uint8_t*)ehdr) + ehdr->e_phoff); + aux[i++].set_long(AT_PHNUM, ehdr->e_phnum); + + // Misc aux[i++].set_ptr(AT_BASE, nullptr); aux[i++].set_long(AT_FLAGS, 0x0); aux[i++].set_ptr(AT_ENTRY, (void*) &kernel_main); aux[i++].set_long(AT_HWCAP, 0); - aux[i++].set_long(AT_UID, 0); aux[i++].set_long(AT_EUID, 0); aux[i++].set_long(AT_GID, 0); @@ -127,22 +231,36 @@ void kernel_start(uintptr_t magic, uintptr_t addr) const char* plat = "x86_64"; aux[i++].set_ptr(AT_PLATFORM, plat); - aux[i++].set_long(AT_NULL, 0); std::array argv; - // arguments to "program" + + // Parameters to main argv[0] = (char*) Service::name(); argv[1] = 0x0; int argc = 1; - // "environment" variables - argv[2] = "BLARGH=0"; + // Env vars + argv[2] = "USER=root"; argv[3] = 0x0; memcpy(&argv[4], aux, sizeof(auxv_t) * 38); + Expects(elf.is_ELF() && "Elf header present"); + +#if defined(__x86_64__) + kprintf("* Initialize syscall MSR\n"); + uint64_t star_kernel_cs = 8ull << 32; + uint64_t star_user_cs = 8ull << 48; + uint64_t star = star_kernel_cs | star_user_cs; + x86::CPU::write_msr(IA32_STAR, star); + x86::CPU::write_msr(IA32_LSTAR, (uintptr_t)&__syscall_entry); +#elif defined(__i386__) + kprintf("Initialize syscall MSR (32)\n"); + #warning Classical syscall interface missing for 32-bit +#endif - // ubp_av = argv, irrelevant when argc = 0 - hallo__libc_start_main(kernel_main, argc, argv.data()); + //GDB_ENTRY; + kprintf("* Starting libc initialization\n"); + __libc_start_main(kernel_main, argc, argv.data()); } diff --git a/src/platform/x86_pc/musl_init.cpp b/src/platform/x86_pc/musl_init.cpp deleted file mode 100644 index 6cb93603d2..0000000000 --- a/src/platform/x86_pc/musl_init.cpp +++ /dev/null @@ -1,155 +0,0 @@ -#include -#include -#include -#include -extern "C" void panic(const char*); - -#ifndef LIBC_H -#define LIBC_H - -#include -#include -#include - -struct __locale_map; - -struct __locale_struct { - const struct __locale_map *volatile cat[6]; -}; - -struct tls_module { - struct tls_module *next; - void *image; - size_t len, size, align, offset; -}; - -struct __libc { - int can_do_threads; - int threaded; - int secure; - volatile int threads_minus_1; - size_t *auxv; - struct tls_module *tls_head; - size_t tls_size, tls_align, tls_cnt; - size_t page_size; - struct __locale_struct global_locale; -}; - -#ifndef PAGE_SIZE -#define PAGE_SIZE libc.page_size -#endif - -#ifdef __PIC__ -#define ATTR_LIBC_VISIBILITY __attribute__((visibility("hidden"))) -#else -#define ATTR_LIBC_VISIBILITY -#endif - -extern struct __libc __libc ATTR_LIBC_VISIBILITY; -#define libc __libc - -extern size_t __hwcap ATTR_LIBC_VISIBILITY; -extern size_t __sysinfo ATTR_LIBC_VISIBILITY; -extern char *__progname, *__progname_full; - -/* Designed to avoid any overhead in non-threaded processes */ -void __lock(volatile int *) ATTR_LIBC_VISIBILITY; -void __unlock(volatile int *) ATTR_LIBC_VISIBILITY; -int __lockfile(FILE *) ATTR_LIBC_VISIBILITY; -void __unlockfile(FILE *) ATTR_LIBC_VISIBILITY; -#define LOCK(x) __lock(x) -#define UNLOCK(x) __unlock(x) - -void __synccall(void (*)(void *), void *); -int __setxid(int, int, int, int); - -extern char **__environ; - -#undef weak_alias -#define weak_alias(old, new) \ - extern __typeof(old) new __attribute__((weak, alias(#old))) - -#undef LFS64_2 -#define LFS64_2(x, y) weak_alias(x, y) - -#undef LFS64 -#define LFS64(x) LFS64_2(x, x##64) - -#endif - - -void __init_tls(size_t *); - -extern "C" void _init(); - -__attribute__((__weak__, __visibility__("hidden"))) -extern void (*const __init_array_start)(void), (*const __init_array_end)(void); - -void __init_ssp(void*) {} - -#define AUX_CNT 38 - -void hallo__init_libc(char **envp, char *pn) -{ - kprintf("__init_libc(%p, %p)\n", envp, pn); - size_t i, *auxv, aux[AUX_CNT] = { 0 }; - __environ = envp; - for (i=0; envp[i]; i++); - libc.auxv = auxv = (size_t *)(envp+i+1); - kprintf("aux: %p\n", auxv); - - for (i=0; auxv[i]; i+=2) if (auxv[i]cpuid); // PER_CPU on gs #else // initialize GDT for this core @@ -199,7 +200,7 @@ namespace x86 // load GDT and refresh segments GDT::reload_gdt(gdt); // enable per-cpu and per-thread - GDT::set_fs(fs); + //GDT::set_fs(fs); GDT::set_gs(gs); #endif // hardware barrier diff --git a/test/kernel/integration/tls/CMakeLists.txt b/test/kernel/integration/tls/CMakeLists.txt index a37bc330a1..30ebde06c6 100644 --- a/test/kernel/integration/tls/CMakeLists.txt +++ b/test/kernel/integration/tls/CMakeLists.txt @@ -1,5 +1,7 @@ cmake_minimum_required(VERSION 2.8.9) +option(threading "" ON) + # IncludeOS install location if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) set(ENV{INCLUDEOS_PREFIX} /usr/local) @@ -19,6 +21,7 @@ set(SOURCES ) set(DRIVERS + boot_logger ) set(PLUGINS From 9f32de48b1a15afa11b16428e755d82998ee954d Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 19:54:46 +0100 Subject: [PATCH 037/723] build: script to build alternative unwinder --- etc/build_libunwind_nongnu.sh | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 etc/build_libunwind_nongnu.sh diff --git a/etc/build_libunwind_nongnu.sh b/etc/build_libunwind_nongnu.sh new file mode 100755 index 0000000000..8f57b605f8 --- /dev/null +++ b/etc/build_libunwind_nongnu.sh @@ -0,0 +1,33 @@ +#! /bin/bash +#. $INCLUDEOS_SRC/etc/set_traps.sh + +# Dependencies +sudo apt install autoconf + +# Download, configure, compile and install llvm +ARCH=${ARCH:-x86_64} # CPU architecture. Alternatively x86_64 +TARGET=$ARCH-elf # Configure target based on arch. Always ELF. +BUILD_DIR=${BUILD_DIR:-~/IncludeOS_build/build_llvm} + +# Download +if [ ! -d libunwind ]; then + echo -e "\n\n >>> Getting libunwind \n" + git clone git://git.sv.gnu.org/libunwind.git +fi + +cd libunwind + +git checkout $libunwind_version +make clean || true + +ls -l + +./autogen.sh + +# NOTE: For some reason target x86_pc-elf doesn't work for libunwind. +./configure --prefix=$TEMP_INSTALL_DIR/$TARGET --disable-shared --enable-debug --target=$ARCH-linux --enable-cxx-exceptions +make $num_jobs +make install + + +#trap - EXIT From 3ad6af06386c39a22c293e1e5049439e029be265 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 26 Jan 2018 20:52:01 +0100 Subject: [PATCH 038/723] linker.ld: add ELF header and keep alignment --- src/arch/x86_64/linker.ld | 48 ++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/src/arch/x86_64/linker.ld b/src/arch/x86_64/linker.ld index a67b45e883..429898ef89 100644 --- a/src/arch/x86_64/linker.ld +++ b/src/arch/x86_64/linker.ld @@ -16,22 +16,25 @@ * See the License for the specific language governing permissions and * limitations under the License. **/ + ENTRY(_start) SECTIONS { - PROVIDE ( _ELF_START_ = . + 0xA00000); - PROVIDE ( _LOAD_START_ = _ELF_START_); /* For convenience w. multiboot */ - . = _ELF_START_; + PROVIDE (__executable_start = SEGMENT_START("text-segment", 0xa00000)); + . = SEGMENT_START("text-segment", 0xa00000) + SIZEOF_HEADERS; + + /* For convenience w. multiboot */ + PROVIDE ( _ELF_START_ = __executable_start); + PROVIDE ( _LOAD_START_ = _ELF_START_ ); .multiboot : { - PROVIDE(_MULTIBOOT_START_ = .); + PROVIDE(_MULTIBOOT_START_ = . ); *(.multiboot) } - - .text ALIGN(0x10): + .text ALIGN(0x1000): { PROVIDE( _TEXT_START_ = . ); /* For solo5, although it's just used to print the mem layout. */ @@ -42,8 +45,8 @@ SECTIONS /* For solo5, although it's just used to print the mem layout. */ _etext = .; } - PROVIDE( _TEXT_END_ = . ); + PROVIDE( _TEXT_END_ = . ); .init ALIGN(0x10) : { _INIT_START_ = .; *(.init) @@ -122,14 +125,7 @@ SECTIONS PROVIDE_HIDDEN (__fini_array_end = .); } - .config ALIGN(0x8) : { - _CONFIG_JSON_START_ = .; - KEEP(*(.config)) - _CONFIG_JSON_END_ = .; - BYTE(0); - } - - .rodata : + .rodata ALIGN(0x1000): { _RODATA_START_ = .; *(.rodata*) @@ -139,16 +135,23 @@ SECTIONS _erodata = .; } - /* For stack unwinding (exception handling) */ - .eh_frame_hdr ALIGN(0x8): - { - KEEP(*(.eh_frame_hdr*)) + .config ALIGN(0x8) : { + _CONFIG_JSON_START_ = .; + KEEP(*(.config)) + _CONFIG_JSON_END_ = .; + BYTE(0); } - .eh_frame ALIGN(0x8): + + .eh_frame : { PROVIDE (__eh_frame_start = .); KEEP(*(.eh_frame)) - LONG (0); + PROVIDE (__eh_frame_end = .); + } + + .eh_frame_hdr : + { + KEEP(*(.eh_frame_hdr)) } .gcc_except_table : @@ -172,6 +175,7 @@ SECTIONS _TDATA_END_ = .; . = ALIGN(0x10); } + .tbss : { _TBSS_START_ = .; @@ -197,6 +201,7 @@ SECTIONS .bss ALIGN(0x1000) : { + _BSS_START_ = .; *(.bss .bss.* .gnu.linkonce.b.*) *(COMMON) @@ -206,6 +211,7 @@ SECTIONS _end = .; + PROVIDE (end = .); PROVIDE (_ELF_END_ = .); PROVIDE (_LOAD_END_ = .); From 58ea8ef5849702241a024ed68f3c2e6e5f165f2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 29 Jan 2018 08:53:40 +0100 Subject: [PATCH 039/723] tcp: SEQ getters for read buffer --- api/net/tcp/read_buffer.hpp | 12 +++++++++++- src/net/tcp/read_buffer.cpp | 7 +------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/api/net/tcp/read_buffer.hpp b/api/net/tcp/read_buffer.hpp index 60e69ea01d..453543a051 100644 --- a/api/net/tcp/read_buffer.hpp +++ b/api/net/tcp/read_buffer.hpp @@ -65,7 +65,11 @@ class Read_buffer { * * @return Amount of bytes that fits in the buffer starting from seq */ - size_t fits(const seq_t seq) const; + size_t fits(const seq_t seq) const + { + const auto rel = (seq - start); + return (rel < capacity()) ? (capacity() - rel) : 0; + } /** * @brief Exposes the internal buffer @@ -151,6 +155,12 @@ class Read_buffer { void set_start(const seq_t seq) { start = seq; } + seq_t start_seq() const + { return start; } + + seq_t end_seq() const + { return start + capacity(); } + int deserialize_from(void*); int serialize_to(void*) const; diff --git a/src/net/tcp/read_buffer.cpp b/src/net/tcp/read_buffer.cpp index bd0b360b0f..dc1c84a71e 100644 --- a/src/net/tcp/read_buffer.cpp +++ b/src/net/tcp/read_buffer.cpp @@ -35,6 +35,7 @@ size_t Read_buffer::insert(const seq_t seq, const uint8_t* data, size_t len, boo size_t rel = seq - start; assert(rel < capacity() && "No point trying to write at or above the end"); + //printf("seq=%u, start=%u, rel: %lu sz=%lu\n", seq, start, rel, size()); // avoid writing above size by shrinking len len = std::min(capacity() - rel, len); @@ -55,12 +56,6 @@ size_t Read_buffer::insert(const seq_t seq, const uint8_t* data, size_t len, boo return len; } -size_t Read_buffer::fits(const seq_t seq) const -{ - const auto rel = (seq - start); - return (rel < capacity()) ? (capacity() - rel) : 0; -} - void Read_buffer::reset(const seq_t seq) { this->reset(seq, buf->capacity()); From 630c938641b657313ff3222e28c3502e45a46566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 31 Jan 2018 12:23:16 +0100 Subject: [PATCH 040/723] tcp: Read_request now supports more than one buffer --- api/net/tcp/connection.hpp | 12 +- api/net/tcp/connection.inc | 7 +- api/net/tcp/read_request.hpp | 40 ++++-- lib/LiveUpdate/serialize_tcp.cpp | 6 +- src/CMakeLists.txt | 3 +- src/net/tcp/connection.cpp | 37 ++--- src/net/tcp/connection_states.cpp | 2 +- src/net/tcp/read_request.cpp | 178 ++++++++++++++++++++++++ test/CMakeLists.txt | 2 + test/net/unit/tcp_read_request_test.cpp | 86 ++++++++++++ 10 files changed, 334 insertions(+), 39 deletions(-) create mode 100644 src/net/tcp/read_request.cpp create mode 100644 test/net/unit/tcp_read_request_test.cpp diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index ee02e92dda..2f8a2c140a 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -462,7 +462,7 @@ class Connection { * @return bytes not yet read */ size_t readq_size() const - { return (read_request) ? read_request->buffer.size() : 0; } + { return (read_request) ? read_request->size() : 0; } /** * @brief Total number of bytes in send queue @@ -807,7 +807,7 @@ class Connection { TCB cb; /** The given read request */ - std::unique_ptr read_request; + std::unique_ptr read_request; /** Queue for write requests to process */ Write_queue writeq; @@ -876,6 +876,14 @@ class Connection { //RTTM::seconds SRTT_prev{1.0f}; //RTTM::seconds RTTVAR_prev{1.0f}; + /** + * @brief Set the Read_request + * + * @param[in] recv_bufsz The receive bufsz + * @param[in] cb The read callback + */ + void _on_read(size_t recv_bufsz, ReadCallback cb); + // Retrieve the associated shared_ptr for a connection, if it exists // Throws out_of_range if it doesn't Connection_ptr retrieve_shared(); diff --git a/api/net/tcp/connection.inc b/api/net/tcp/connection.inc index 3a095637a5..6f796007c4 100644 --- a/api/net/tcp/connection.inc +++ b/api/net/tcp/connection.inc @@ -10,12 +10,7 @@ inline Connection& Connection::on_connect(ConnectCallback cb) { inline Connection& Connection::on_read(size_t recv_bufsz, ReadCallback cb) { - if(not read_request) { - read_request = std::make_unique(recv_bufsz, seq_t(this->cb.RCV.NXT), cb); - } else { - read_request->buffer.reset(seq_t(this->cb.RCV.NXT), recv_bufsz); - read_request->callback = cb; - } + _on_read(recv_bufsz, cb); return *this; } diff --git a/api/net/tcp/read_request.hpp b/api/net/tcp/read_request.hpp index e7f6378f50..a5c69eb4c8 100644 --- a/api/net/tcp/read_request.hpp +++ b/api/net/tcp/read_request.hpp @@ -1,6 +1,6 @@ // This file is a part of the IncludeOS unikernel - www.includeos.org // -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences +// Copyright 2015-2018 Oslo and Akershus University College of Applied Sciences // and Alfred Bratterud // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,24 +21,44 @@ #include "read_buffer.hpp" #include +#include namespace net { namespace tcp { -struct ReadRequest { +class Read_request { +public: + using Buffer_ptr = std::unique_ptr; using ReadCallback = delegate; - - Read_buffer buffer; + static constexpr size_t buffer_limit = 2; ReadCallback callback; - ReadRequest(const size_t n, const seq_t seq, ReadCallback cb) - : buffer{n, seq}, - callback{cb} - {} + Read_request(size_t size, seq_t start, ReadCallback cb); + + size_t insert(seq_t seq, const uint8_t* data, size_t n, bool psh = false); + + size_t fits(const seq_t seq) const; + + size_t size() const; + + void set_start(seq_t seq); + + void reset(size_t size, const seq_t seq); + + const Read_buffer& front() const + { return *buffers.front(); } + + Read_buffer& front() + { return *buffers.front(); } + +private: + std::deque buffers; + + Read_buffer* get_buffer(const seq_t seq); }; -} // < namespace tcp -} // < namespace net +} +} #endif // < NET_TCP_READ_REQUEST_HPP diff --git a/lib/LiveUpdate/serialize_tcp.cpp b/lib/LiveUpdate/serialize_tcp.cpp index d327d6844d..eb90893925 100644 --- a/lib/LiveUpdate/serialize_tcp.cpp +++ b/lib/LiveUpdate/serialize_tcp.cpp @@ -176,8 +176,8 @@ void Connection::deserialize_from(void* addr) auto* readq = (read_buffer*) &area->vla[writeq_len]; if (readq->capacity) { - read_request = std::make_unique(readq->capacity, readq->seq, nullptr); - read_request->buffer.deserialize_from(readq); + read_request = std::make_unique(readq->capacity, readq->seq, nullptr); + read_request->front().deserialize_from(readq); } if (area->rtx_is_running) { @@ -270,7 +270,7 @@ int Connection::serialize_to(void* addr) const /// serialize read queue auto* readq = (read_buffer*) &area->vla[writeq_len]; - int readq_len = (read_request) ? read_request->buffer.serialize_to(readq) : sizeof(read_buffer); + int readq_len = (read_request) ? read_request->front().serialize_to(readq) : sizeof(read_buffer); //printf("READ: %u SEND: %u REMAIN: %u STATE: %s\n", // readq_size(), sendq_size(), sendq_remaining(), cb.to_string().c_str()); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3cf3e1fbcf..ae78ca8f9f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -45,7 +45,8 @@ set(OS_OBJECTS net/ethernet/ethernet.cpp net/ethernet/ethernet_8021q.cpp net/checksum.cpp net/ip4/arp.cpp net/ip4/ip4.cpp net/ip4/reassembly.cpp net/tcp/tcp.cpp net/tcp/connection.cpp net/tcp/connection_states.cpp - net/tcp/write_queue.cpp net/tcp/rttm.cpp net/tcp/listener.cpp net/tcp/read_buffer.cpp + net/tcp/write_queue.cpp net/tcp/rttm.cpp net/tcp/listener.cpp + net/tcp/read_buffer.cpp net/tcp/read_request.cpp net/ip4/icmp4.cpp net/ip4/udp.cpp net/ip4/udp_socket.cpp net/dns/dns.cpp net/dns/client.cpp net/dhcp/dh4client.cpp net/dhcp/dhcpd.cpp net/buffer_store.cpp net/inet4.cpp diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index a7c9fae549..2e4aab328f 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -62,6 +62,23 @@ Connection::~Connection() rtx_clear(); } +void Connection::_on_read(size_t recv_bufsz, ReadCallback cb) +{ + if(read_request == nullptr) + { + read_request = std::make_unique(recv_bufsz, seq_t(this->cb.RCV.NXT), cb); + } + // read request is already set, only reset if new size. + else + { + read_request->callback = cb; + // this will flush the current data to the user (if any) + read_request->reset(recv_bufsz, seq_t(this->cb.RCV.NXT)); + // TODO: we should probably clear any SACK if it exists, + // due to reset() throwing away buffers + } +} + Connection_ptr Connection::retrieve_shared() { return host_.retrieve_shared(this); } @@ -112,27 +129,15 @@ size_t Connection::receive(seq_t seq, const uint8_t* data, size_t n, bool PUSH) // should not be called without an read request Expects(read_request); Expects(n > 0); - - auto& buf = read_request->buffer; size_t received{0}; while(n) { - auto read = buf.insert(seq, data + received, n, PUSH); + auto read = read_request->insert(seq, data + received, n, PUSH); n -= read; // subtract amount of data left to insert received += read; // add to the total recieved seq += read; // advance the sequence number - - // deliver if finished for delivery - if(buf.is_ready()) - { - if (read_request->callback) - read_request->callback(buf.buffer()); - - // reset/clear readbuffer (new sequence start) - buf.reset(seq); - } } return received; @@ -266,7 +271,7 @@ void Connection::close() { } void Connection::receive_disconnect() { - Expects(read_request and read_request->buffer.buffer()); + Expects(read_request and read_request->size()); if(read_request->callback) { // TODO: consider adding back when SACK is complete @@ -700,7 +705,7 @@ void Connection::recv_data(const Packet& in) // Keep track if a packet is being sent during the async read callback const auto snd_nxt = cb.SND.NXT; - if(read_request and read_request->buffer.capacity()) + if(read_request) { auto recv = receive(in.seq(), in.tcp_data(), length, in.isset(PSH)); Ensures(recv == length); @@ -971,7 +976,7 @@ void Connection::signal_connect(const bool success) // if on read was set before we got a seq number, // update the starting sequence number for the read buffer if(read_request and success) - read_request->buffer.set_start(cb.RCV.NXT); + read_request->set_start(cb.RCV.NXT); if(on_connect_) (success) ? on_connect_(retrieve_shared()) : on_connect_(nullptr); diff --git a/src/net/tcp/connection_states.cpp b/src/net/tcp/connection_states.cpp index 6ff844ba84..e4445638d3 100644 --- a/src/net/tcp/connection_states.cpp +++ b/src/net/tcp/connection_states.cpp @@ -330,7 +330,7 @@ void Connection::State::process_fin(Connection& tcp, const Packet& in) { //tcb.RCV.NXT += fin; const auto snd_nxt = tcb.SND.NXT; // empty the read buffer - if(tcp.read_request and tcp.read_request->buffer.buffer()) + if(tcp.read_request and tcp.read_request->size()) tcp.receive_disconnect(); // signal disconnect to the user tcp.signal_disconnect(Disconnect::CLOSING); diff --git a/src/net/tcp/read_request.cpp b/src/net/tcp/read_request.cpp new file mode 100644 index 0000000000..ba1bcfd362 --- /dev/null +++ b/src/net/tcp/read_request.cpp @@ -0,0 +1,178 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +namespace net { +namespace tcp { + + Read_request::Read_request(size_t size, seq_t start, ReadCallback cb) + : callback{cb} + { + buffers.push_back(std::make_unique(size, start)); + } + + size_t Read_request::insert(seq_t seq, const uint8_t* data, size_t n, bool psh) + { + Expects(not buffers.empty()); + + //printf("insert: seq=%u len=%lu\n", seq, n); + size_t recv{0}; + while(n) + { + auto* buf = get_buffer(seq); + + if(UNLIKELY(buf == nullptr)) + break; + + //printf("got buffer start=%u end=%u missing=%lu\n", + // buf->start_seq(), buf->end_seq(), buf->missing()); + + auto read = buf->insert(seq, data + recv, n, psh); + + n -= read; // subtract amount of data left to insert + recv += read; // add to the total recieved + seq += read; // advance the sequence number + + // flush this (and other buffers) if ready + // (and also being the one in the front) + while(buf->is_ready() and buf == buffers.front().get()) + { + callback(buf->buffer()); + + // this is the only one, so we can reuse it + if(buffers.size() == 1) + { + buf->reset(seq); + break; + } + // to make it simple, just get rid of it and have the + // algo create a new one if needed + // TODO: reuse it by putting in in the back + // (need to check if there is room for more, and + // maybe it isnt even necessary..) + else + { + buffers.pop_front(); + buf = buffers.front().get(); + } + } + } + + Ensures(not buffers.empty()); + return recv; + } + + Read_buffer* Read_request::get_buffer(const seq_t seq) + { + // There is room in a existing one + for(auto& ptr : buffers) + { + if(ptr->fits(seq) > 0) + return ptr.get(); + } + + // We can still create a new one + if(buffers.size() < buffer_limit) + { + // current cap + const auto& back = buffers.back(); + + // TODO: if the gap is bigger than 1 buffer + // we probably need to create multiple buffers, + // ... or just decide we only support gaps of 1 buffer size. + buffers.push_back( + std::make_unique(back->capacity(), back->end_seq())); + + //printf("new buffer added,fits(%lu)=%lu\n", + // seq, buffers.back()->fits(seq)); + + if(buffers.back()->fits(seq) > 0) + return buffers.back().get(); + } + + return nullptr; + } + + size_t Read_request::fits(const seq_t seq) const + { + // There is room in a existing one + for(auto& ptr : buffers) + { + if(ptr->fits(seq) > 0) + return ptr->fits(seq); + } + + // Possible to create one (or more) to support this? + if(buffers.size() < buffer_limit) + { + auto& back = buffers.back(); + const auto rel = seq - back->end_seq(); + const auto cap = back->capacity(); + + if(rel < cap) + return (cap - rel); + } + + return 0; + } + + size_t Read_request::size() const + { + size_t bytes = 0; + + for(auto& ptr : buffers) + bytes += ptr->size(); + + return bytes; + } + + void Read_request::set_start(seq_t seq) + { + for(auto& ptr : buffers) + { + Expects(ptr->size() == 0 && "Cannot change start sequence when there already is data"); + ptr->set_start(seq); + seq += ptr->capacity(); + Ensures(seq == ptr->end_seq()); + } + } + + void Read_request::reset(size_t size, const seq_t seq) + { + Expects(not buffers.empty()); + + auto it = buffers.begin(); + + // get the first buffer + auto* buf = it->get(); + // if it contains data without any holes, + // return it to the user + if(buf->size() > 0 and buf->missing() == 0) + { + callback(buf->buffer()); + } + // reset the first buffer + buf->reset(seq, size); + // throw the others away + buffers.erase(++it, buffers.end()); + + Ensures(buffers.size() == 1); + } + +} +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3094caa01e..c3308673eb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -99,6 +99,7 @@ set(TEST_SOURCES ${TEST}/net/unit/super_stack.cpp ${TEST}/net/unit/tcp_packet_test.cpp ${TEST}/net/unit/tcp_read_buffer_test.cpp + ${TEST}/net/unit/tcp_read_request_test.cpp ${TEST}/net/unit/tcp_write_queue.cpp ${TEST}/net/unit/router_test.cpp ${TEST}/posix/unit/fd_map_test.cpp @@ -184,6 +185,7 @@ set(OS_SOURCES ${SRC}/net/tcp/rttm.cpp ${SRC}/net/tcp/tcp.cpp ${SRC}/net/tcp/read_buffer.cpp + ${SRC}/net/tcp/read_request.cpp ${SRC}/net/tcp/write_queue.cpp ${SRC}/posix/fcntl.cpp ${SRC}/posix/fd.cpp diff --git a/test/net/unit/tcp_read_request_test.cpp b/test/net/unit/tcp_read_request_test.cpp new file mode 100644 index 0000000000..39c3777e59 --- /dev/null +++ b/test/net/unit/tcp_read_request_test.cpp @@ -0,0 +1,86 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +CASE("Operating with out of order data") +{ + using namespace net::tcp; + const size_t BUFSZ = 4096; + const seq_t SEQ_START = 0; + const size_t SEGSZ = 1460; + + uint8_t data[SEGSZ]; + seq_t seq = SEQ_START; + int no_reads; + size_t fits, insert; + + Read_request::ReadCallback read_cb = [&](auto buf) mutable { + no_reads++; + }; + + auto req = std::make_unique(BUFSZ, seq, read_cb); + no_reads = 0; + + // Insert hole, first missing + fits = req->fits(seq + SEGSZ); + EXPECT(fits == BUFSZ - SEGSZ); + insert = req->insert(seq + SEGSZ, data, std::min(SEGSZ, fits)); + EXPECT(insert == SEGSZ); + + // Insert in order + fits = req->fits(seq); + EXPECT(fits == BUFSZ); + insert = req->insert(seq, data, std::min(SEGSZ, fits)); + EXPECT(insert == SEGSZ); + // Increase sequence number + seq += insert; + seq += SEGSZ; // for the one added earlier + + // Insert 2 ahead of current pos (result in new buffer) (with PUSH) + fits = req->fits(seq + (SEGSZ*2)); + EXPECT(fits == BUFSZ - ((SEGSZ*4) - BUFSZ)); // ... + insert = req->insert(seq + (SEGSZ*2), data, std::min(SEGSZ, fits), true); + EXPECT(insert == SEGSZ); + + // In order, filling the last bytes in the first buffer + fits = req->fits(seq); + EXPECT(fits == BUFSZ - (SEGSZ*2)); + insert = req->insert(seq, data, std::min(SEGSZ, fits)); + EXPECT(insert == fits); + EXPECT(no_reads == 1); // first buffer should be cleared + seq += insert; // add the inserted bytes + + auto remaining = SEGSZ - insert; + EXPECT(remaining < SEGSZ); + fits = req->fits(seq); + EXPECT(fits == BUFSZ); + insert = req->insert(seq, data, remaining); + EXPECT(insert == remaining); + seq += insert; + + // Fill the missing packet + fits = req->fits(seq); + EXPECT(fits == BUFSZ - remaining); + insert = req->insert(seq, data, std::min(SEGSZ, fits)); + EXPECT(no_reads == 2); + seq += insert; + seq += SEGSZ; // for the one added earlier + + EXPECT(seq == (uint32_t)(SEQ_START + SEGSZ*5)); +} From 2186f7ab48e94891ac47dc0083c4919bbe824ee0 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 6 Feb 2018 10:04:36 +0100 Subject: [PATCH 041/723] Merge dev/musl --- api/arch.hpp | 1 + api/hw/ioport.hpp | 27 ++ api/kernel/os.hpp | 24 +- cmake/cross_compiled_libraries.txt | 4 +- diskimagebuild/filetree.cpp | 34 +- etc/vboxrun.sh | 2 +- lib/uplink/ws_uplink.cpp | 11 +- linux/src/os.cpp | 8 +- linux/userspace/CMakeLists.txt | 2 +- seed/service/CMakeLists.txt | 2 + seed/service/service.cpp | 1 + seed/service/vm.json | 4 + src/CMakeLists.txt | 1 - src/arch/x86_64/arch_start.asm | 50 +-- src/arch/x86_64/linker.ld | 4 +- src/chainload/service.cpp | 7 +- src/drivers/CMakeLists.txt | 5 +- src/drivers/e1000.cpp | 325 ++++++++++++++++++ src/drivers/e1000.hpp | 138 ++++++++ src/drivers/e1000_defs.hpp | 107 ++++++ src/kernel/cpuid.cpp | 1 - src/kernel/multiboot.cpp | 6 +- src/kernel/os.cpp | 2 +- src/kernel/syscalls.cpp | 5 - src/net/buffer_store.cpp | 4 +- src/platform/kvm/pv_eoi.cpp | 54 +++ src/platform/x86_linux/CMakeLists.txt | 15 - src/platform/x86_linux/kernel_start.cpp | 67 ---- src/platform/x86_linux/os.cpp | 180 ---------- src/platform/x86_linux/platform.cpp | 72 ---- src/platform/x86_linux/sanity_checks.cpp | 83 ----- src/platform/x86_linux/serial1.cpp | 14 - src/platform/x86_linux/syscall.hpp | 48 --- src/platform/x86_pc/apic.cpp | 79 ++--- src/platform/x86_pc/os.cpp | 26 +- src/platform/x86_pc/platform.cpp | 2 +- src/platform/x86_pc/smbios.cpp | 40 +++ src/platform/x86_pc/softreset.cpp | 2 +- src/platform/x86_pc/start.asm | 9 +- src/platform/x86_solo5/os.cpp | 9 - test/CMakeLists.txt | 4 - .../kernel/integration/LiveUpdate/service.cpp | 4 +- .../integration/LiveUpdate/test_boot.cpp | 18 +- .../kernel/integration/modules/CMakeLists.txt | 1 - test/kernel/integration/modules/hotswap.cpp | 10 +- .../integration/modules/mod2/CMakeLists.txt | 3 - test/kernel/integration/modules/service.cpp | 25 +- test/kernel/integration/timers/timers.cpp | 7 +- test/lest_util/os_mock.cpp | 17 +- vmrunner/vm.schema.json | 5 + vmrunner/vmrunner.py | 7 +- 51 files changed, 906 insertions(+), 670 deletions(-) create mode 100644 seed/service/vm.json create mode 100644 src/drivers/e1000.cpp create mode 100644 src/drivers/e1000.hpp create mode 100644 src/drivers/e1000_defs.hpp delete mode 100644 src/platform/x86_linux/CMakeLists.txt delete mode 100644 src/platform/x86_linux/kernel_start.cpp delete mode 100644 src/platform/x86_linux/os.cpp delete mode 100644 src/platform/x86_linux/platform.cpp delete mode 100644 src/platform/x86_linux/sanity_checks.cpp delete mode 100644 src/platform/x86_linux/serial1.cpp delete mode 100644 src/platform/x86_linux/syscall.hpp diff --git a/api/arch.hpp b/api/arch.hpp index f5c5347101..8b5a7d26c5 100644 --- a/api/arch.hpp +++ b/api/arch.hpp @@ -63,6 +63,7 @@ inline void __sw_barrier() noexcept struct arch_system_info_t { std::string uuid; + uint64_t physical_memory; }; const arch_system_info_t& __arch_system_info() noexcept; diff --git a/api/hw/ioport.hpp b/api/hw/ioport.hpp index 5708a2d802..887cb0d520 100644 --- a/api/hw/ioport.hpp +++ b/api/hw/ioport.hpp @@ -74,6 +74,33 @@ namespace hw { asm volatile ("outw %%ax,%%dx"::"a" (data), "d"(port)); #else #error "outw() not implemented for selected arch" +#endif + } + + /** Receive a double-word from port. + @param port : The port number to receive from + */ + static inline uint32_t inl(int port) + { + uint32_t ret; +#if defined(ARCH_x86) + //asm volatile ("xorl %eax,%eax"); + asm volatile ("inl %%dx,%%eax":"=a" (ret):"d"(port)); +#else +#error "inw() not implemented for selected arch" +#endif + return ret; + } + + /** Send a double-word to port. + @param port : The port to send to + @param data : Double-word of data + */ + static inline void outl(int port, uint32_t data) { +#if defined(ARCH_x86) + asm volatile ("outl %%eax,%%dx"::"a" (data), "d"(port)); +#else +#error "outw() not implemented for selected arch" #endif } diff --git a/api/kernel/os.hpp b/api/kernel/os.hpp index 5b72b5d810..cb0b4eb32f 100644 --- a/api/kernel/os.hpp +++ b/api/kernel/os.hpp @@ -60,17 +60,16 @@ class OS { static const char* cmdline_args() noexcept; /** Clock cycles since boot. */ - static uint64_t cycles_since_boot() { - return __arch_cpu_cycles(); - } - /** Nanoseconds since boot */ + static uint64_t cycles_since_boot() noexcept; + + /** Nanoseconds since boot converted from cycles */ static uint64_t nanos_since_boot() noexcept; /** Timestamp for when OS was booted */ - static RTC::timestamp_t boot_timestamp(); + static RTC::timestamp_t boot_timestamp() noexcept; /** Uptime in whole seconds. */ - static RTC::timestamp_t uptime(); + static RTC::timestamp_t uptime() noexcept; /** Time spent sleeping (halt) in cycles */ static uint64_t cycles_asleep() noexcept; @@ -296,11 +295,20 @@ inline OS::Span_mods OS::modules() return nullptr; } -inline RTC::timestamp_t OS::boot_timestamp() +inline uint64_t OS::cycles_since_boot() noexcept +{ + return __arch_cpu_cycles(); +} +inline uint64_t OS::nanos_since_boot() noexcept +{ + return (cycles_since_boot() * 1e6) / cpu_freq().count(); +} + +inline RTC::timestamp_t OS::boot_timestamp() noexcept { return RTC::boot_timestamp(); } -inline RTC::timestamp_t OS::uptime() +inline RTC::timestamp_t OS::uptime() noexcept { return RTC::time_since_boot(); } diff --git a/cmake/cross_compiled_libraries.txt b/cmake/cross_compiled_libraries.txt index 03bd9b7813..92c159750d 100644 --- a/cmake/cross_compiled_libraries.txt +++ b/cmake/cross_compiled_libraries.txt @@ -18,8 +18,8 @@ else(BUNDLE_LOC) include(ExternalProject) ExternalProject_Add(PrecompiledLibraries PREFIX precompiled - URL https://github.com/hioa-cs/IncludeOS/releases/download/v0.11.0-bundle/IncludeOS_dependencies_v0-11-0-rc-1.tar.gz - URL_HASH SHA1=f719886bcab5c55957361c024bab54c601d862f5 + URL https://github.com/hioa-cs/IncludeOS/releases/download/v0.12.0-rc.2/IncludeOS_dependencies_v0-12-0_musl_libunwind.tar.gz + URL_HASH SHA1=4c275a797c1417ef2a2b5cbedc0f947dd7b31f7d CONFIGURE_COMMAND "" BUILD_COMMAND "" UPDATE_COMMAND "" diff --git a/diskimagebuild/filetree.cpp b/diskimagebuild/filetree.cpp index 08e20bd5f2..6b846bf0de 100644 --- a/diskimagebuild/filetree.cpp +++ b/diskimagebuild/filetree.cpp @@ -96,6 +96,10 @@ void FileSys::add_dir(Dir& dvec) fprintf(stderr, "Unable to open directory %s\n", cwd_buffer); throw std::runtime_error("Unable to open directory " + std::string(cwd_buffer)); } + + std::vector sub_dirs; + std::vector sub_files; + struct dirent* ent; while ((ent = readdir(dir)) != nullptr) { @@ -103,17 +107,33 @@ void FileSys::add_dir(Dir& dvec) if (name == ".." || name == ".") continue; if (ent->d_type == DT_DIR) { - auto& d = dvec.add_dir(ent->d_name); - add_dir(d); + sub_dirs.push_back(std::move(name)); } else { - try { - dvec.add_file(ent->d_name); - } catch (std::exception& e) { - fprintf(stderr, "%s\n", e.what()); - } + sub_files.push_back(std::move(name)); + } + } + // close directory before adding more folders and files + res = closedir(dir); + if (res < 0) { + throw std::runtime_error("diskbuilder: Failed to close directory"); + } + + // add sub directories + for (const auto& dirname : sub_dirs) { + auto& d = dvec.add_dir(dirname.c_str()); + add_dir(d); + } + // add files in current directory + for (const auto& filename : sub_files) + { + try { + dvec.add_file(filename.c_str()); + } catch (std::exception& e) { + fprintf(stderr, "%s\n", e.what()); } } + // pop work dir res = chdir(pwd_buffer); if (res < 0) { diff --git a/etc/vboxrun.sh b/etc/vboxrun.sh index cf6a4695fe..75df4aec64 100755 --- a/etc/vboxrun.sh +++ b/etc/vboxrun.sh @@ -79,7 +79,7 @@ else # Creating and registering the VM and adding a virtual IDE drive to it, # then attaching the hdd image. echo -e "\nCreating VM: $VMNAME ...\n" - $VB createvm --name "$VMNAME" --ostype "Other" --register + $VB createvm --name "$VMNAME" --ostype "Other_64" --register $VB storagectl "$VMNAME" --name "IDE Controller" --add ide --bootable on $VB storageattach "$VMNAME" --storagectl "IDE Controller" --port 0 --device 0 --type 'hdd' --medium "$targetLoc" diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index 1d34d85728..14e6420164 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -97,7 +97,7 @@ namespace uplink { // BINARY HASH store.add_string(0, binary_hash_); // nanos timestamp of when update begins - store.add (1, OS::cycles_since_boot()); + store.add (1, OS::nanos_since_boot()); } void WS_uplink::restore(liu::Restore& store) @@ -106,10 +106,8 @@ namespace uplink { binary_hash_ = store.as_string(); store.go_next(); // calculate update cycles taken - uint64_t cycles = store.as_type (); store.go_next(); - cycles = OS::cycles_since_boot() - cycles; - // cycles to nanos - this->update_time_taken = cycles / (OS::cpu_freq().count() / 1.0e6); + uint64_t prev_nanos = store.as_type (); store.go_next(); + this->update_time_taken = OS::nanos_since_boot() - prev_nanos; INFO2("Update took %.3f millis", this->update_time_taken / 1.0e6); } @@ -395,6 +393,9 @@ namespace uplink { writer.Key("arch"); writer.String(OS::arch()); + writer.Key("physical_ram"); + writer.Uint64(sysinfo.physical_memory); + // CPU Features auto features = CPUID::detect_features_str(); writer.Key("cpu_features"); diff --git a/linux/src/os.cpp b/linux/src/os.cpp index 9e559d6ab2..795d32a777 100644 --- a/linux/src/os.cpp +++ b/linux/src/os.cpp @@ -5,12 +5,18 @@ #include #include #include -uint64_t OS::nanos_since_boot() noexcept +uint64_t __arch_system_time() noexcept { struct timespec tv; clock_gettime(CLOCK_REALTIME, &tv); return tv.tv_sec*(uint64_t)1000000000ull+tv.tv_nsec; } +timespec __arch_wall_clock() noexcept +{ + struct timespec tv; + clock_gettime(CLOCK_REALTIME, &tv); + return tv; +} void OS::event_loop() { diff --git a/linux/userspace/CMakeLists.txt b/linux/userspace/CMakeLists.txt index 72e3275b79..69bef9499f 100644 --- a/linux/userspace/CMakeLists.txt +++ b/linux/userspace/CMakeLists.txt @@ -50,7 +50,7 @@ set(NET_SOURCES ) if (CUSTOM_BOTAN) set(NET_SOURCES ${NET_SOURCES} - "${IOS}/src/net/http/secure_server.cpp") + "${IOS}/src/net/https/botan_server.cpp") endif() set(OS_SOURCES diff --git a/seed/service/CMakeLists.txt b/seed/service/CMakeLists.txt index c199102f0e..e898549165 100644 --- a/seed/service/CMakeLists.txt +++ b/seed/service/CMakeLists.txt @@ -24,6 +24,8 @@ set(DRIVERS # virtionet # Virtio networking # virtioblock # Virtio block device # ... Others from IncludeOS/src/drivers + + #boot_logger # Enable lots of logging from boot stage ) set(PLUGINS diff --git a/seed/service/service.cpp b/seed/service/service.cpp index eff58d76d2..eff95fd30b 100644 --- a/seed/service/service.cpp +++ b/seed/service/service.cpp @@ -22,4 +22,5 @@ void Service::start(const std::string& args) { printf("Hello world - OS included!\n"); printf("Args = %s\n", args.c_str()); + printf("Try giving the service less memory, eg. 5MB in vm.json\n"); } diff --git a/seed/service/vm.json b/seed/service/vm.json new file mode 100644 index 0000000000..023195a1eb --- /dev/null +++ b/seed/service/vm.json @@ -0,0 +1,4 @@ +{ + "net" : [], + "mem" : 64 +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9b08510b56..de3fa743a1 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -71,7 +71,6 @@ set_source_files_properties(crt/cxx_abi.cpp PROPERTIES COMPILE_FLAGS "-fno-sanit add_subdirectory(arch/${ARCH}) add_subdirectory(platform/x86_pc) add_subdirectory(platform/x86_nano) -add_subdirectory(platform/x86_linux) if(WITH_SOLO5) add_subdirectory(platform/x86_solo5) endif(WITH_SOLO5) diff --git a/src/arch/x86_64/arch_start.asm b/src/arch/x86_64/arch_start.asm index 7e9b9946d3..2a52c91a91 100644 --- a/src/arch/x86_64/arch_start.asm +++ b/src/arch/x86_64/arch_start.asm @@ -24,7 +24,7 @@ extern __multiboot_addr %define P4_TAB 0x1000 %define P3_TAB 0x2000 ;; - 0x5fff %define P2_TAB 0x100000 -%define STACK_LOCATION 0x9ffff0 +%define STACK_LOCATION 0x9D3F0 %define EFER.LME 0x100 %define EFER.SCE 0x1 @@ -37,6 +37,8 @@ extern __multiboot_addr %define IA32_GS_BASE 0xc0000101 %define IA32_KERNEL_GS_BASE 0xc0000102 +%define NUM_P3_ENTRIES 5 +%define NUM_P2_ENTRIES 2560 [BITS 32] __arch_start: @@ -48,7 +50,8 @@ __arch_start: ;; address for Page Map Level 4 mov edi, P4_TAB mov cr3, edi - mov ecx, 0x3000 / 0x4 + ;; clear out P4 and P3 + mov ecx, 0x2000 / 0x4 xor eax, eax ; Nullify the A-register. rep stosd @@ -56,32 +59,33 @@ __arch_start: mov edi, P4_TAB mov DWORD [edi], P3_TAB | 0x3 ;; present+write - ;; create 4x page directory pointer table entries + ;; create 1GB mappings + mov ecx, NUM_P3_ENTRIES mov edi, P3_TAB - mov ebx, P2_TAB | 0x3 ;; present + write - mov DWORD [edi], ebx + mov eax, P2_TAB | 0x3 ;; present + write + mov ebx, 0x0 + +.p3_loop: + mov DWORD [edi], eax ;; Low word + mov DWORD [edi+4], ebx ;; High word + add eax, 1 << 12 ;; page increments + adc ebx, 0 ;; increment high word when CF set add edi, 8 - add ebx, 0x1000 - mov DWORD [edi], ebx - add edi, 8 - add ebx, 0x1000 - mov DWORD [edi], ebx - add edi, 8 - add ebx, 0x1000 - mov DWORD [edi], ebx + loop .p3_loop - ;; create page directory entries - mov ecx, 512*4 ;; num entries + ;; create 2MB mappings + mov ecx, NUM_P2_ENTRIES mov edi, P2_TAB - - ;; start at address 0x0 - mov ebx, 0x0 | 0x3 | 1 << 7 ;; present+write + huge -.ptd_loop: - mov DWORD [edi], ebx ;; Assign the physical adress to lower 32-bits - mov DWORD [edi+4], 0x0 ;; Zero out the rest of the 64-bit word - add ebx, 1 << 21 ;; 2MB increments + mov eax, 0x0 | 0x3 | 1 << 7 ;; present + write + huge + mov ebx, 0x0 + +.p2_loop: + mov DWORD [edi], eax ;; Low word + mov DWORD [edi+4], ebx ;; High word + add eax, 1 << 21 ;; 2MB increments + adc ebx, 0 ;; increment high word when CF set add edi, 8 - loop .ptd_loop + loop .p2_loop ;; enable PAE mov eax, cr4 diff --git a/src/arch/x86_64/linker.ld b/src/arch/x86_64/linker.ld index 429898ef89..967815e188 100644 --- a/src/arch/x86_64/linker.ld +++ b/src/arch/x86_64/linker.ld @@ -22,8 +22,8 @@ ENTRY(_start) SECTIONS { - PROVIDE (__executable_start = SEGMENT_START("text-segment", 0xa00000)); - . = SEGMENT_START("text-segment", 0xa00000) + SIZEOF_HEADERS; + PROVIDE (__executable_start = SEGMENT_START("text-segment", 0x100000)); + . = SEGMENT_START("text-segment", 0x100000) + SIZEOF_HEADERS; /* For convenience w. multiboot */ PROVIDE ( _ELF_START_ = __executable_start); diff --git a/src/chainload/service.cpp b/src/chainload/service.cpp index ff3c726faa..2015882c2f 100644 --- a/src/chainload/service.cpp +++ b/src/chainload/service.cpp @@ -20,6 +20,7 @@ #include #include #include +#include extern bool os_enable_boot_logging; @@ -40,8 +41,10 @@ void promote_mod_to_kernel() Expects (bootinfo->mods_count); auto* mod = (multiboot_module_t*)bootinfo->mods_addr; - // Set command line param to mod param - bootinfo->cmdline = mod->cmdline; + // Move commandline to a relatively safe area + const uintptr_t RELATIVELY_SAFE_AREA = 0x8000; + strcpy((char*) RELATIVELY_SAFE_AREA, (const char*) mod->cmdline); + bootinfo->cmdline = RELATIVELY_SAFE_AREA; // Subtract one module (bootinfo->mods_count)--; diff --git a/src/drivers/CMakeLists.txt b/src/drivers/CMakeLists.txt index 5e2be9517f..b9c982af49 100644 --- a/src/drivers/CMakeLists.txt +++ b/src/drivers/CMakeLists.txt @@ -24,6 +24,9 @@ add_dependencies(virtionet PrecompiledLibraries) add_library(vmxnet3 STATIC vmxnet3.cpp) add_dependencies(vmxnet3 PrecompiledLibraries) +add_library(e1000 STATIC e1000.cpp) +add_dependencies(e1000 PrecompiledLibraries) + add_library(ip4_reassembly STATIC "ip4_reassembly.cpp") add_dependencies(ip4_reassembly PrecompiledLibraries) @@ -45,7 +48,7 @@ add_dependencies(vga_output PrecompiledLibraries) install(TARGETS ide_readwrite ide_readonly ide_writeonly virtionet virtioblk - vmxnet3 + vmxnet3 e1000 ip4_reassembly heap_debugging boot_logger disk_logger disklog_reader diff --git a/src/drivers/e1000.cpp b/src/drivers/e1000.cpp new file mode 100644 index 0000000000..2f2a8a7fdc --- /dev/null +++ b/src/drivers/e1000.cpp @@ -0,0 +1,325 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "e1000.hpp" +#include "e1000_defs.hpp" +#include +#include +#include +#include +#include +#include +// loosely based on OSdev article http://wiki.osdev.org/Intel_Ethernet_i217 + +static int deferred_event = 0; +static std::vector deferred_devices; + +e1000::e1000(hw::PCI_Device& d) : + Link(Link_protocol{{this, &e1000::transmit}, mac()}, bufstore_), + m_pcidev(d), bufstore_{1024, 2048} +{ + INFO("e1000", "Intel Pro/1000 Ethernet Adapter (rev=%#x)", d.rev_id()); + + // legacy IRQ from PCI + uint32_t value = d.read_dword(PCI::CONFIG_INTR); + this->m_irq = value & 0xFF; + assert(this->m_irq != 0xFF); + + Events::get().subscribe(this->m_irq, {this, &e1000::event_handler}); + __arch_enable_legacy_irq(this->m_irq); + INFO2("Subscribed on IRQ %u", this->m_irq); + + if (deferred_event == 0) + { + deferred_event = Events::get().subscribe(&e1000::do_deferred_xmit); + } + + // shared-memory & I/O address + this->shm_base = d.get_bar(0); + this->io_base = d.iobase(); + + // initialize + write_cmd(REG_CTRL, (1 << 26)); + + this->link_up(); + + // have to clear out the multicast filter, otherwise shit breaks + for(int i = 0; i < 128; i++) + write_cmd(0x5200 + i*4, 0); + for(int i = 0; i < 64; i++) + write_cmd(0x4000 + i*4, 0); + + /* Disables flow control */ + write_cmd(0x0028, 0); + write_cmd(0x002c, 0); + write_cmd(0x0030, 0); + write_cmd(0x0170, 0); + + this->intr_enable(); + + // initialize RX + for (int i = 0; i < NUM_RX_DESC; i++) { + rx.desc[i].addr = (uint64_t) new_rx_packet(); + rx.desc[i].status = 0; + } + + uint64_t rx_desc_ptr = (uint64_t) rx.desc; + write_cmd(REG_RXDESCLO, rx_desc_ptr); + write_cmd(REG_RXDESCHI, 0); + write_cmd(REG_RXDESCLEN, NUM_RX_DESC * sizeof(rx_desc)); + write_cmd(REG_RXDESCHEAD, 0); + write_cmd(REG_RXDESCTAIL, NUM_RX_DESC-1); + +#define BROADCAST_ENABLE 0x8000 +#define STRIP_ETH_CRC 0x4000000 +#define RX_BUFFER_2048 0x0 + uint32_t rx_flags = RX_BUFFER_2048 | STRIP_ETH_CRC | BROADCAST_ENABLE + | (1 << 5) | (0 << 8) | (0 << 4) | (0 << 3) | ( 1 << 2); + write_cmd(REG_RCTRL, rx_flags); + + // initialize TX + memset(tx.desc, 0, sizeof(tx.desc)); + for (int i = 0; i < NUM_TX_DESC; i++) { + tx.desc[i].status = 0x1; // done + } + + uint64_t tx_desc_ptr = (uint64_t) tx.desc; + write_cmd(REG_TXDESCLO, tx_desc_ptr); + write_cmd(REG_TXDESCHI, 0); + write_cmd(REG_TXDESCLEN, NUM_TX_DESC * sizeof(tx_desc)); + write_cmd(REG_TXDESCHEAD, 0); + write_cmd(REG_TXDESCTAIL, NUM_TX_DESC); + + write_cmd(REG_TCTRL, (1 << 1) | (1 << 3)); + + // get MAC address + this->retrieve_hw_addr(); + + // GO! + uint32_t flags = read_cmd(REG_RCTRL); + write_cmd(REG_RCTRL, flags | RCTL_EN); + // verify device status + assert(read_cmd(REG_STATUS) == 0x80080783); +} + +uint32_t e1000::read_cmd(uint16_t cmd) +{ + //hw::outl(this->io_base, cmd); + //return hw::inl(this->io_base + 4); + return *(volatile uint32_t*) (this->shm_base + cmd); +} +void e1000::write_cmd(uint16_t cmd, uint32_t val) +{ + //hw::outl(this->io_base, cmd); + //hw::outl(this->io_base + 4, val); + *(volatile uint32_t*) (this->shm_base + cmd) = val; +} + +void e1000::retrieve_hw_addr() +{ + auto* mac_src = (const char*) (this->shm_base + 0x5400); + memcpy(&this->hw_addr, mac_src, sizeof(hw_addr)); + INFO2("MAC address: %s", hw_addr.to_string().c_str()); +} + +void e1000::link_up() +{ + uint32_t flags = read_cmd(REG_CTRL); + write_cmd(REG_CTRL, flags | ECTRL_SLU); + + int success = (read_cmd(REG_STATUS) & (1 << 1)) != 0; + INFO("e1000", "Link up: %s", (success) ? "true" : "false"); +} + +void e1000::intr_enable() +{ + write_cmd(REG_IMASK, 0x1F6DC); + write_cmd(REG_IMASK, 0xFF & ~4); + read_cmd(0xC0); +} + +net::Packet_ptr +e1000::recv_packet(uint8_t* data, uint16_t size) +{ + auto* ptr = (net::Packet*) (data - DRIVER_OFFSET - sizeof(net::Packet)); + new (ptr) net::Packet( + DRIVER_OFFSET, + size, + DRIVER_OFFSET + packet_len(), + &bufstore()); + return net::Packet_ptr(ptr); +} +net::Packet_ptr +e1000::create_packet(int link_offset) +{ + auto buffer = bufstore().get_buffer(); + auto* ptr = (net::Packet*) buffer.addr; + new (ptr) net::Packet( + DRIVER_OFFSET + link_offset, + 0, + DRIVER_OFFSET + packet_len(), + buffer.bufstore); + return net::Packet_ptr(ptr); +} +uintptr_t e1000::new_rx_packet() +{ + auto* pkt = bufstore().get_buffer().addr; + return (uintptr_t) &pkt[sizeof(net::Packet) + DRIVER_OFFSET]; +} + +void e1000::event_handler() +{ + uint32_t status = read_cmd(0xC0); + // see: e1000_regs.h + //printf("e1000: event %x received\n", status); + + // empty transmit queue + if (status & 0x02) + { + //printf("tx queue empty!\n"); + if (sendq) { + transmit(std::move(sendq)); + } + if (can_transmit()) { + transmit_queue_available_event(NUM_TX_DESC); + } + } + // link status change + if (status & 0x04) + { + this->link_up(); + } + if (status & 0x40) + { + printf("rx overrun!\n"); + } + // rx timer interrupt + if (status & 0x80) + { + recv_handler(); + } +} + +void e1000::recv_handler() +{ + uint16_t old_idx = 0xffff; + + while (rx.desc[rx.current].status & 1) + { + auto& tk = rx.desc[rx.current]; + auto* buf = (uint8_t*) tk.addr; + + //printf("e1000: recv %u bytes\n", tk.length); + auto pkt = recv_packet(buf, tk.length); + Link_layer::receive(std::move(pkt)); + + // give new buffer + tk.addr = (uint64_t) this->new_rx_packet(); + tk.status = 0; + // go to next index + old_idx = rx.current; + rx.current = (rx.current + 1) % NUM_RX_DESC; + } + if (old_idx != 0xffff) + write_cmd(REG_RXDESCTAIL, old_idx); +} + +void e1000::transmit(net::Packet_ptr pckt) +{ + if (sendq == nullptr) + sendq = std::move(pckt); + else + sendq->chain(std::move(pckt)); + // send as much as possible from sendq + while (sendq != nullptr && can_transmit()) + { + auto next = sendq->detach_tail(); + // transmit released buffer + auto* packet = sendq.release(); + transmit_data(packet->buf() + DRIVER_OFFSET, packet->size()); + // next is the new sendq + sendq = std::move(next); + } +} +bool e1000::can_transmit() +{ + return (tx.desc[tx.current].status & 0xFF) == 0x1; +} +void e1000::transmit_data(uint8_t* data, uint16_t length) +{ + auto& tk = tx.desc[tx.current]; + assert(tk.status == 0x1 && "Descriptor must be done"); + + if (tk.addr != 0x0) { + auto* packet = (net::Packet*) (tk.addr - DRIVER_OFFSET - sizeof(net::Packet)); + delete packet; // call deleter on Packet to release it + } + //printf("e1000: xmit %p -> %u bytes\n", data, length); + tk.addr = (uint64_t) data; + tk.length = length; + tk.cmd = (1 << 3) | 0x3; + tk.status = 0; + + tx.current = (tx.current + 1) % NUM_TX_DESC; + if (tx.deferred == false) + { + tx.deferred = true; + deferred_devices.push_back(this); + Events::get().trigger_event(deferred_event); + } +} +void e1000::xmit_kick() +{ + write_cmd(REG_TXDESCTAIL, tx.current); + tx.deferred = false; +} +void e1000::do_deferred_xmit() +{ + for (auto& dev : deferred_devices) + dev->xmit_kick(); + deferred_devices.clear(); +} + +void e1000::flush() +{ + this->transmit(std::move(sendq)); +} +void e1000::poll() +{ + this->recv_handler(); +} +void e1000::deactivate() +{ + uint32_t flags = read_cmd(REG_RCTRL); + write_cmd(REG_RCTRL, flags & ~RCTL_EN); +} +void e1000::move_to_this_cpu() +{ + // TODO: implement me +} + +#include +__attribute__((constructor)) +static void register_func() +{ + PCI_manager::register_nic(PCI::VENDOR_INTEL, 0x109A, &e1000::new_instance); + PCI_manager::register_nic(PCI::VENDOR_INTEL, 0x100E, &e1000::new_instance); + PCI_manager::register_nic(PCI::VENDOR_INTEL, 0x100F, &e1000::new_instance); + PCI_manager::register_nic(PCI::VENDOR_INTEL, 0x153A, &e1000::new_instance); + PCI_manager::register_nic(PCI::VENDOR_INTEL, 0x1539, &e1000::new_instance); + PCI_manager::register_nic(PCI::VENDOR_INTEL, 0x10EA, &e1000::new_instance); +} diff --git a/src/drivers/e1000.hpp b/src/drivers/e1000.hpp new file mode 100644 index 0000000000..ecf1443d5e --- /dev/null +++ b/src/drivers/e1000.hpp @@ -0,0 +1,138 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +class e1000 : public net::Link_layer +{ +public: + using Link = net::Link_layer; + using Link_protocol = Link::Protocol; + static const int DRIVER_OFFSET = 2; + static const int NUM_TX_DESC = 64; + static const int NUM_RX_DESC = 128; + + static std::unique_ptr new_instance(hw::PCI_Device& d) + { return std::make_unique(d); } + + const char* driver_name() const override { + return "e1000"; + } + + const MAC::Addr& mac() const noexcept override { + return this->hw_addr; + } + + uint16_t MTU() const noexcept override { + return 1500; + } + + uint16_t packet_len() const noexcept { + return sizeof(net::ethernet::Header) + MTU(); + } + + net::downstream create_physical_downstream() override + { return {this, &e1000::transmit}; } + + net::Packet_ptr create_packet(int) override; + + size_t frame_offset_device() override + { return DRIVER_OFFSET; }; + + /** Linklayer input. Hooks into IP-stack bottom, w.DOWNSTREAM data.*/ + void transmit(net::Packet_ptr pckt); + + /** Constructor. @param pcidev an initialized PCI device. */ + e1000(hw::PCI_Device& pcidev); + + /** Space available in the transmit queue, in packets */ + size_t transmit_queue_available() override { + return 1; + } + + void flush() override; + + void deactivate() override; + + void move_to_this_cpu() override; + + void poll() override; + +private: + void intr_enable(); + void intr_disable(); + void link_up(); + void retrieve_hw_addr(); + + uint32_t read_cmd(uint16_t cmd); + void write_cmd(uint16_t cmd, uint32_t val); + + net::Packet_ptr recv_packet(uint8_t*, uint16_t); + uintptr_t new_rx_packet(); + void event_handler(); + void recv_handler(); + bool can_transmit(); + void transmit_data(uint8_t*, uint16_t); + void xmit_kick(); + static void do_deferred_xmit(); + + hw::PCI_Device& m_pcidev; + std::vector irqs; + uint16_t io_base; + uintptr_t shm_base; + MAC::Addr hw_addr; + + uint8_t m_irq; + + struct rx_desc + { + uint64_t addr; + uint16_t length; + uint16_t checksum; + uint8_t status; + uint8_t errors; + uint16_t special; + } __attribute__((packed, aligned(16))); + struct tx_desc + { + uint64_t addr; + uint16_t length; + uint8_t cso; + uint8_t cmd; + uint8_t status; + uint8_t css; + uint16_t special; + } __attribute__((packed, aligned(16))); + + struct rx_t { + rx_desc desc[NUM_RX_DESC]; + uint16_t current = 0; + } rx; + + struct tx_t { + tx_desc desc[NUM_TX_DESC]; + uint16_t current = 0; + bool deferred = false; + } tx; + + // sendq as packet chain + net::Packet_ptr sendq = nullptr; + net::BufferStore bufstore_; +}; diff --git a/src/drivers/e1000_defs.hpp b/src/drivers/e1000_defs.hpp new file mode 100644 index 0000000000..ab51f3a6a6 --- /dev/null +++ b/src/drivers/e1000_defs.hpp @@ -0,0 +1,107 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +// see: http://wiki.osdev.org/Intel_Ethernet_i217 + +#define REG_CTRL 0x0000 +#define REG_STATUS 0x0008 +#define REG_EEPROM 0x0014 +#define REG_CTRL_EXT 0x0018 +#define REG_IMASK 0x00D0 +#define REG_RCTRL 0x0100 +#define REG_RXDESCLO 0x2800 +#define REG_RXDESCHI 0x2804 +#define REG_RXDESCLEN 0x2808 +#define REG_RXDESCHEAD 0x2810 +#define REG_RXDESCTAIL 0x2818 + +#define REG_TCTRL 0x0400 +#define REG_TXDESCLO 0x3800 +#define REG_TXDESCHI 0x3804 +#define REG_TXDESCLEN 0x3808 +#define REG_TXDESCHEAD 0x3810 +#define REG_TXDESCTAIL 0x3818 + + +#define REG_RDTR 0x2820 // RX Delay Timer Register +#define REG_RXDCTL 0x3828 // RX Descriptor Control +#define REG_RADV 0x282C // RX Int. Absolute Delay Timer +#define REG_RSRPD 0x2C00 // RX Small Packet Detect Interrupt + + + +#define REG_TIPG 0x0410 // Transmit Inter Packet Gap +#define ECTRL_SLU 0x40 //set link up + + +#define RCTL_EN (1 << 1) // Receiver Enable +#define RCTL_SBP (1 << 2) // Store Bad Packets +#define RCTL_UPE (1 << 3) // Unicast Promiscuous Enabled +#define RCTL_MPE (1 << 4) // Multicast Promiscuous Enabled +#define RCTL_LPE (1 << 5) // Long Packet Reception Enable +#define RCTL_LBM_NONE (0 << 6) // No Loopback +#define RCTL_LBM_PHY (3 << 6) // PHY or external SerDesc loopback +#define RTCL_RDMTS_HALF (0 << 8) // Free Buffer Threshold is 1/2 of RDLEN +#define RTCL_RDMTS_QUARTER (1 << 8) // Free Buffer Threshold is 1/4 of RDLEN +#define RTCL_RDMTS_EIGHTH (2 << 8) // Free Buffer Threshold is 1/8 of RDLEN +#define RCTL_MO_36 (0 << 12) // Multicast Offset - bits 47:36 +#define RCTL_MO_35 (1 << 12) // Multicast Offset - bits 46:35 +#define RCTL_MO_34 (2 << 12) // Multicast Offset - bits 45:34 +#define RCTL_MO_32 (3 << 12) // Multicast Offset - bits 43:32 +#define RCTL_BAM (1 << 15) // Broadcast Accept Mode +#define RCTL_VFE (1 << 18) // VLAN Filter Enable +#define RCTL_CFIEN (1 << 19) // Canonical Form Indicator Enable +#define RCTL_CFI (1 << 20) // Canonical Form Indicator Bit Value +#define RCTL_DPF (1 << 22) // Discard Pause Frames +#define RCTL_PMCF (1 << 23) // Pass MAC Control Frames +#define RCTL_SECRC (1 << 26) // Strip Ethernet CRC + +// Buffer Sizes +#define RCTL_BSIZE_256 (3 << 16) +#define RCTL_BSIZE_512 (2 << 16) +#define RCTL_BSIZE_1024 (1 << 16) +#define RCTL_BSIZE_2048 (0 << 16) +#define RCTL_BSIZE_4096 ((3 << 16) | (1 << 25)) +#define RCTL_BSIZE_8192 ((2 << 16) | (1 << 25)) +#define RCTL_BSIZE_16384 ((1 << 16) | (1 << 25)) + + +// Transmit Command + +#define CMD_EOP (1 << 0) // End of Packet +#define CMD_IFCS (1 << 1) // Insert FCS +#define CMD_IC (1 << 2) // Insert Checksum +#define CMD_RS (1 << 3) // Report Status +#define CMD_RPS (1 << 4) // Report Packet Sent +#define CMD_VLE (1 << 6) // VLAN Packet Enable +#define CMD_IDE (1 << 7) // Interrupt Delay Enable + + +// TCTL Register + +#define TCTL_EN (1 << 1) // Transmit Enable +#define TCTL_PSP (1 << 3) // Pad Short Packets +#define TCTL_CT_SHIFT 4 // Collision Threshold +#define TCTL_COLD_SHIFT 12 // Collision Distance +#define TCTL_SWXOFF (1 << 22) // Software XOFF Transmission +#define TCTL_RTLC (1 << 24) // Re-transmit on Late Collision + +#define TSTA_DD (1 << 0) // Descriptor Done +#define TSTA_EC (1 << 1) // Excess Collisions +#define TSTA_LC (1 << 2) // Late Collision +#define LSTA_TU (1 << 3) // Transmit Underrun diff --git a/src/kernel/cpuid.cpp b/src/kernel/cpuid.cpp index 21b9dbda0c..6de96cfd4c 100644 --- a/src/kernel/cpuid.cpp +++ b/src/kernel/cpuid.cpp @@ -20,7 +20,6 @@ #include #include #include -#include namespace std { diff --git a/src/kernel/multiboot.cpp b/src/kernel/multiboot.cpp index 76a541c9dd..85e24ca16a 100644 --- a/src/kernel/multiboot.cpp +++ b/src/kernel/multiboot.cpp @@ -114,9 +114,9 @@ void OS::multiboot(uint32_t boot_addr) } if (bootinfo_->flags & MULTIBOOT_INFO_CMDLINE) { - INFO2("* Booted with parameters @ 0x%x: %s", bootinfo_->cmdline, - reinterpret_cast(bootinfo_->cmdline)); - OS::cmdline = reinterpret_cast(bootinfo_->cmdline); + const auto* cmdline = (const char*) (uintptr_t) bootinfo_->cmdline; + INFO2("* Booted with parameters @ %p: %s", cmdline, cmdline); + OS::cmdline = strdup(cmdline); } if (bootinfo_->flags & MULTIBOOT_INFO_MEM_MAP) { diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 3b9027e84e..ccad75e891 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -51,7 +51,7 @@ bool OS::m_block_drivers_ready = false; KHz OS::cpu_khz_ {-1}; uintptr_t OS::liveupdate_loc_ = 0; uintptr_t OS::memory_end_ = 0; -uintptr_t OS::heap_max_ = (uintptr_t) -1; +uintptr_t OS::heap_max_ = (uintptr_t)0xffffffffffff; const uintptr_t OS::elf_binary_size_ {(uintptr_t)&_ELF_END_ - (uintptr_t)&_ELF_START_}; // stdout redirection diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index b8b920967e..004bc2ee37 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -48,11 +48,6 @@ char** environ {__env}; extern uintptr_t heap_begin; extern uintptr_t heap_end; -extern "C" -void abort() { - kprintf("Abooooooring!\n"); - panic("abort() called"); -} void _exit(int status) { kprintf("%s",std::string(LINEWIDTH, '=').c_str()); diff --git a/src/net/buffer_store.cpp b/src/net/buffer_store.cpp index e0d8564df7..180f5097ed 100644 --- a/src/net/buffer_store.cpp +++ b/src/net/buffer_store.cpp @@ -116,9 +116,9 @@ namespace net { parent = parent->next_; if (!parent->available_.empty()) return parent; } + BSD_BUF(" Allocating %lu new buffers (%lu total)\n", + local_buffers(), total_buffers() + local_buffers()); parent->next_ = new BufferStore(local_buffers(), bufsize()); - BSD_BUF(" Allocating %lu new buffers (%lu total)", - local_buffers(), total_buffers()); return parent->next_; #else return nullptr; diff --git a/src/platform/kvm/pv_eoi.cpp b/src/platform/kvm/pv_eoi.cpp index e69de29bb2..1ac79d4f62 100644 --- a/src/platform/kvm/pv_eoi.cpp +++ b/src/platform/kvm/pv_eoi.cpp @@ -0,0 +1,54 @@ +#include +#include +#include "../x86_pc/cpu.hpp" +#include + +// *** manual *** +// http://choon.net/forum/read.php?21,1123399 +// https://www.kernel.org/doc/Documentation/virtual/kvm/cpuid.txt + +#define KVM_MSR_ENABLED 1 +#define MSR_KVM_PV_EOI_EN 0x4b564d04 +#define KVM_PV_EOI_BIT 0 +#define KVM_PV_EOI_MASK (0x1 << KVM_PV_EOI_BIT) +#define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK + +// current selected EOI method +extern void (*current_eoi_mechanism)(); +extern void (*real_eoi_mechanism)(); +extern void (*current_intr_handler)(); + +__attribute__ ((aligned(4))) +static volatile unsigned long kvm_exitless_eoi = 0; + +extern "C" +void kvm_pv_eoi() +{ + if (kvm_exitless_eoi) { + kprintf("avoidable eoi\n"); + } + uint8_t reg; + asm("btr %2, %0; setc %1" : "+m"(kvm_exitless_eoi), "=rm"(reg) : "r"(0)); + if (reg) { + kprintf("avoidable eoi\n"); + return; + } + else { + //kprintf("unavoidable eoi\n"); + } + // fallback to normal x2APIC EOI + real_eoi_mechanism(); +} +void kvm_pv_eoi_init() +{ + uint64_t addr = (uint64_t) &kvm_exitless_eoi; + addr |= KVM_MSR_ENABLED; + x86::CPU::write_msr(MSR_KVM_PV_EOI_EN, addr); + // verify that the feature was enabled + uint64_t res = x86::CPU::read_msr(MSR_KVM_PV_EOI_EN); + if (res & 1) { + INFO("KVM", "Paravirtual EOI enabled"); + // set new EOI handler + current_eoi_mechanism = kvm_pv_eoi; + } +} diff --git a/src/platform/x86_linux/CMakeLists.txt b/src/platform/x86_linux/CMakeLists.txt deleted file mode 100644 index 515cc6c95d..0000000000 --- a/src/platform/x86_linux/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# -# x86 userspace Linux platform -# -set(PLATFORM_OBJECTS - os.cpp - serial1.cpp - platform.cpp - kernel_start.cpp - sanity_checks.cpp - ) - -add_library(x86_linux STATIC ${PLATFORM_OBJECTS}) -add_dependencies(x86_linux PrecompiledLibraries) -set_target_properties(x86_linux PROPERTIES LINKER_LANGUAGE CXX) -install(TARGETS x86_linux DESTINATION includeos/${ARCH}/platform) diff --git a/src/platform/x86_linux/kernel_start.cpp b/src/platform/x86_linux/kernel_start.cpp deleted file mode 100644 index 298ee83233..0000000000 --- a/src/platform/x86_linux/kernel_start.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -extern "C" { - void __init_serial1(); - void __init_sanity_checks(); - void kernel_sanity_checks(); - uintptr_t _multiboot_free_begin(uintptr_t boot_addr); - uintptr_t _move_symbols(uintptr_t loc); - void _init_bss(); - void _init_heap(uintptr_t); - void _init_c_runtime(); - void _init_syscalls(); -} - -extern "C" -void kernel_start(uintptr_t magic, uintptr_t addr) -{ - // Determine where free memory starts - extern char _end; - uintptr_t free_mem_begin = reinterpret_cast(&_end); - - if (magic == MULTIBOOT_BOOTLOADER_MAGIC) { - free_mem_begin = _multiboot_free_begin(addr); - } - - // Preserve symbols from the ELF binary - free_mem_begin += _move_symbols(free_mem_begin); - - // Initialize zero-initialized vars - _init_bss(); - - // Initialize heap - _init_heap(free_mem_begin); - - // Initialize stack-unwinder, call global constructors etc. - _init_c_runtime(); - - // Initialize system calls - _init_syscalls(); - - // Initialize early OS, platform and devices - OS::start(magic, addr); - - // Initialize common subsystems and call Service::start - OS::post_start(); - - // Starting event loop from here allows us to profile OS::start - OS::event_loop(); -} diff --git a/src/platform/x86_linux/os.cpp b/src/platform/x86_linux/os.cpp deleted file mode 100644 index 3634716d86..0000000000 --- a/src/platform/x86_linux/os.cpp +++ /dev/null @@ -1,180 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//#define DEBUG -#define MYINFO(X,...) INFO("Kernel", X, ##__VA_ARGS__) - -#include -#include -#include -#include -#include -#include -#include -#include - -//#define ENABLE_PROFILERS -#ifdef ENABLE_PROFILERS -#include -#define PROFILE(name) ScopedProfiler __CONCAT(sp, __COUNTER__){name}; -#else -#define PROFILE(name) /* name */ -#endif - -extern "C" void* get_cpu_esp(); -extern "C" void __libc_init_array(); -extern uintptr_t heap_begin; -extern uintptr_t heap_end; -extern uintptr_t _start; -extern uintptr_t _end; -extern uintptr_t _ELF_START_; -extern uintptr_t _TEXT_START_; -extern uintptr_t _LOAD_START_; -extern uintptr_t _ELF_END_; - -struct alignas(SMP_ALIGN) OS_CPU { - uint64_t cycles_hlt = 0; -}; -static SMP_ARRAY os_per_cpu; - -uint64_t OS::nanos_since_boot() noexcept { - return cycles_since_boot() / cpu_freq().count(); -} - -uint64_t OS::cycles_asleep() noexcept { - return PER_CPU(os_per_cpu).cycles_hlt; -} -uint64_t OS::nanos_asleep() noexcept { - return PER_CPU(os_per_cpu).cycles_hlt / cpu_freq().count(); -} - -__attribute__((noinline)) -void OS::halt() -{ - uint64_t cycles_before = __arch_cpu_cycles(); - asm volatile("hlt"); - - // add a global symbol here so we can quickly discard - // event loop from stack sampling - asm volatile( - ".global _irq_cb_return_location;\n" - "_irq_cb_return_location:" ); - - // Count sleep cycles - PER_CPU(os_per_cpu).cycles_hlt += __arch_cpu_cycles() - cycles_before; -} - -void OS::default_stdout(const char* str, const size_t len) -{ - __serial_print(str, len); -} - -void OS::start(uint32_t boot_magic, uint32_t boot_addr) -{ - OS::cmdline = Service::binary_name(); - // Initialize stdout handlers - OS::add_stdout(&OS::default_stdout); - - PROFILE("OS::start"); - // Print a fancy header - CAPTION("#include // Literally"); - - MYINFO("Stack: %p", get_cpu_esp()); - MYINFO("Boot magic: 0x%x, addr: 0x%x", boot_magic, boot_addr); - - /// STATMAN /// - PROFILE("Statman"); - /// initialize on page 7, 3 pages in size - Statman::get().init(0x6000, 0x3000); - - // Call global ctors - PROFILE("Global constructors"); - __libc_init_array(); - - // BOOT METHOD // - PROFILE("Multiboot / legacy"); - OS::memory_end_ = 0; - // Detect memory limits etc. depending on boot type - if (boot_magic == MULTIBOOT_BOOTLOADER_MAGIC) { - OS::multiboot(boot_addr); - } else { - - if (is_softreset_magic(boot_magic) && boot_addr != 0) - OS::resume_softreset(boot_addr); - - OS::legacy_boot(); - } - assert(OS::memory_end_ != 0); - // Give the rest of physical memory to heap - OS::heap_max_ = OS::memory_end_; - - PROFILE("Memory map"); - // Assign memory ranges used by the kernel - auto& memmap = memory_map(); - MYINFO("Assigning fixed memory ranges (Memory map)"); - - memmap.assign_range({0x6000, 0x8fff, "Statman", "Statistics"}); -#if defined(ARCH_x86_64) - memmap.assign_range({0x100000, 0x8fffff, "Pagetables", "System page tables"}); - memmap.assign_range({0x900000, 0x9fffff, "Stack", "System main stack"}); -#elif defined(ARCH_i686) - memmap.assign_range({0xA000, 0x9fbff, "Stack", "System main stack"}); -#endif - memmap.assign_range({(uintptr_t)&_LOAD_START_, (uintptr_t)&_end - 1, - "ELF", "Your service binary including OS"}); - - assert(::heap_begin != 0x0 and OS::heap_max_ != 0x0); - // @note for security we don't want to expose this - memmap.assign_range({(uintptr_t)&_end, ::heap_begin - 1, - "Pre-heap", "Heap randomization area"}); - - uintptr_t span_max = std::numeric_limits::max(); - uintptr_t heap_range_max_ = std::min(span_max, OS::heap_max_); - - MYINFO("Assigning heap"); - memmap.assign_range({::heap_begin, heap_range_max_, - "Heap", "Dynamic memory", heap_usage }); - - MYINFO("Printing memory map"); - for (const auto &i : memmap) - INFO2("* %s",i.second.to_string().c_str()); - - - PROFILE("Platform init"); - extern void __platform_init(); - __platform_init(); - - PROFILE("RTC init"); - // Realtime/monotonic clock - RTC::init(); -} - -void OS::event_loop() -{ - Events::get(0).process_events(); - do { - OS::halt(); - Events::get(0).process_events(); - } while (power_); - - MYINFO("Stopping service"); - Service::stop(); - - MYINFO("Powering off"); - extern void __arch_poweroff(); - __arch_poweroff(); -} diff --git a/src/platform/x86_linux/platform.cpp b/src/platform/x86_linux/platform.cpp deleted file mode 100644 index 5d77084700..0000000000 --- a/src/platform/x86_linux/platform.cpp +++ /dev/null @@ -1,72 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#define MYINFO(X,...) INFO("x86", X, ##__VA_ARGS__) - -extern "C" char* get_cpu_esp(); -extern "C" void* get_cpu_ebp(); -#define _SENTINEL_VALUE_ 0x123456789ABCDEF - -namespace tls { - extern size_t get_tls_size(); - extern void fill_tls_data(char*); -} -struct alignas(64) smp_table -{ - // thread self-pointer - void* tls_data; // 0x0 - // per-cpu cpuid (and more) - int cpuid; - int reserved; - -#ifdef ARCH_x86_64 - uintptr_t pad[3]; // 64-bit padding - uintptr_t guard; // _SENTINEL_VALUE_ -#else - uint32_t pad[2]; - uintptr_t guard; // _SENTINEL_VALUE_ -#endif - /** put more here **/ -}; -#ifdef ARCH_x86_64 -// FS:0x28 on Linux is storing a special sentinel stack-guard value -static_assert(offsetof(smp_table, guard) == 0x28, "Linux stack sentinel"); -#endif - -void __platform_init() -{ - INFO("Linux", "Initialize event manager"); - Events::get(0).init_local(); - -} - -void __arch_enable_legacy_irq(uint8_t) {} -void __arch_disable_legacy_irq(uint8_t) {} - -void __arch_poweroff() -{ - // exit(0) syscall - __builtin_unreachable(); -} -void __arch_reboot() -{ - // exit(0) syscall - __builtin_unreachable(); -} diff --git a/src/platform/x86_linux/sanity_checks.cpp b/src/platform/x86_linux/sanity_checks.cpp deleted file mode 100644 index 959471f473..0000000000 --- a/src/platform/x86_linux/sanity_checks.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include - -// NOTE: crc_to MUST NOT be initialized to zero -static uint32_t crc_ro = CRC32_BEGIN(); -const auto* LOW_CHECK_SIZE = (volatile int*) 0x200; - -// Global constructors -static int gconstr_value = 0; -__attribute__((constructor)) -static void self_test_gconstr() { - gconstr_value = 1; -} - -static uint32_t generate_ro_crc() noexcept -{ - extern char _TEXT_START_; - extern char _TEXT_END_; - extern char _RODATA_START_; - extern char _RODATA_END_; - return crc32_fast(&_TEXT_START_, &_RODATA_END_ - &_TEXT_START_); -} - -extern "C" -void __init_sanity_checks() noexcept -{ - // zero low memory - for (volatile int* lowmem = NULL; lowmem < LOW_CHECK_SIZE; lowmem++) - *lowmem = 0; - // generate checksum for read-only portions of kernel - crc_ro = generate_ro_crc(); -} - -extern "C" -void kernel_sanity_checks() -{ - // verify checksum of read-only portions of kernel - uint32_t new_ro = generate_ro_crc(); - if (crc_ro != new_ro) { - kprintf("CRC mismatch %#x vs %#x\n", crc_ro, new_ro); - panic("Sanity checks: CRC of kernel read-only area failed"); - } - // verify that first page is zeroes only - for (volatile int* lowmem = NULL; lowmem < LOW_CHECK_SIZE; lowmem++) - if (UNLIKELY(*lowmem != 0)) { - kprintf("Memory at %p was not zeroed: %#x\n", lowmem, *lowmem); - panic("Sanity checks: Low-memory zero test"); - } - - // verify that Elf symbols were not overwritten - bool symbols_verified = Elf::verify_symbols(); - if (!symbols_verified) - panic("Sanity checks: Consistency of Elf symbols and string areas"); - - // global constructor self-test - if (gconstr_value != 1) { - kprintf("Sanity checks: Global constructors not working (or modified during run-time)!\n"); - panic("Sanity checks: Global constructors verification failed"); - } - -} diff --git a/src/platform/x86_linux/serial1.cpp b/src/platform/x86_linux/serial1.cpp deleted file mode 100644 index ef25e33b1f..0000000000 --- a/src/platform/x86_linux/serial1.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include - -extern "C" -void __serial_print1(const char* cstr) -{ - size_t len = strlen(cstr); - write(0, cstr, len); -} -extern "C" -void __serial_print(const char* str, size_t len) -{ - write(0, str, len); -} diff --git a/src/platform/x86_linux/syscall.hpp b/src/platform/x86_linux/syscall.hpp deleted file mode 100644 index fb26bf9bf8..0000000000 --- a/src/platform/x86_linux/syscall.hpp +++ /dev/null @@ -1,48 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -#include -#include - -#define SYS_WRITE 1 - - -#define SYSCALL_1(num, arg1) \ - asm ("movq %0, %%rdi" : : "r"(arg1)); \ - asm ("movq %0, %%rax" : : "i"(num)); \ - asm ("syscall; movl %0, %%eax" : "=r"(result)); - -#define SYSCALL_2(num, arg1, arg2) \ - asm ("movq %0, %%rdi" : : "r"(arg1)); \ - asm ("movq %0, %%rsi" : : "r"(arg2)); \ - asm ("movq %0, %%rax" : : "i"(num)); \ - asm ("syscall; movl %0, %%eax" : "=r"(result)); - -#define SYSCALL_3(num, arg1, arg2, arg3) \ - asm ("mov %0, %%rdi" : : "r"(arg1)); \ - asm ("mov %0, %%rsi" : : "r"(arg2)); \ - asm ("mov %0, %%rdx" : : "r"(arg3)); \ - asm ("mov %0, %%rax" : : "i"(num)); \ - asm ("syscall; movl %0, %%eax" : "=r"(result)); - -int syscall_write(unsigned int fd, const char* buf, size_t count) -{ - int result; - SYSCALL_3(SYS_WRITE, fd, buf, count); - return result; -} diff --git a/src/platform/x86_pc/apic.cpp b/src/platform/x86_pc/apic.cpp index f0d4352f67..5e3b44f05d 100644 --- a/src/platform/x86_pc/apic.cpp +++ b/src/platform/x86_pc/apic.cpp @@ -23,10 +23,10 @@ #include "smp.hpp" #include #include -#include -#include #include #include +//#define ENABLE_KVM_PV_EOI +//#define ENABLE_DYNAMIC_EOI namespace x86 { @@ -38,10 +38,9 @@ namespace x86 extern "C" { // current selected EOI method - void (*current_eoi_mechanism)(); - void (*current_intr_handler)(); - // KVM para PV-EOI feature - void kvm_pv_eoi(); + void (*current_eoi_mechanism)() = nullptr; + void (*real_eoi_mechanism)() = nullptr; + void (*current_intr_handler)() = nullptr; // shortcut that avoids virtual call void x2apic_send_eoi() { x86::CPU::write_msr(x86::x2apic::BASE_MSR + x2APIC_EOI, 0); @@ -52,18 +51,28 @@ extern "C" { uint8_t vector = x86::APIC::get_isr(); //assert(vector >= IRQ_BASE && vector < 160); Events::get().trigger_event(vector - IRQ_BASE); +#ifdef ENABLE_DYNAMIC_EOI + assert(current_eoi_mechanism != nullptr); + current_eoi_mechanism(); +#else lapic_send_eoi(); +#endif } void x2apic_intr_handler() { uint8_t vector = x86::x2apic::static_get_isr(); //assert(vector >= IRQ_BASE && vector < 160); Events::get().trigger_event(vector - IRQ_BASE); +#ifdef ENABLE_DYNAMIC_EOI + assert(current_eoi_mechanism != nullptr); + current_eoi_mechanism(); +#else x2apic_send_eoi(); +#endif } } -void kvm_pv_eoi_init(); +extern void kvm_pv_eoi_init(); namespace x86 { @@ -75,26 +84,31 @@ namespace x86 if (CPUID::has_feature(CPUID::Feature::X2APIC)) { current_apic = &x2apic::get(); - current_eoi_mechanism = x2apic_send_eoi; + real_eoi_mechanism = x2apic_send_eoi; current_intr_handler = x2apic_intr_handler; } else { // an x86 PC without APIC is insane assert(CPUID::has_feature(CPUID::Feature::APIC) && "If this fails, the machine is insane"); current_apic = &xapic::get(); - current_eoi_mechanism = lapic_send_eoi; + real_eoi_mechanism = lapic_send_eoi; current_intr_handler = xapic_intr_handler; } + if (current_eoi_mechanism == nullptr) + current_eoi_mechanism = real_eoi_mechanism; + // enable xAPIC/x2APIC on this cpu current_apic->enable(); // initialize I/O APICs IOAPIC::init(ACPI::get_ioapics()); +#ifdef ENABLE_KVM_PV_EOI // use KVMs paravirt EOI if supported - //if (CPUID::kvm_feature(KVM_FEATURE_PV_EOI)) - // kvm_pv_eoi_init(); + if (CPUID::kvm_feature(KVM_FEATURE_PV_EOI)) + kvm_pv_eoi_init(); +#endif } void APIC::enable_irq(uint8_t irq) @@ -129,46 +143,3 @@ namespace x86 IOAPIC::disable(irq); } } - -// *** manual *** -// http://choon.net/forum/read.php?21,1123399 -// https://www.kernel.org/doc/Documentation/virtual/kvm/cpuid.txt - -#define KVM_MSR_ENABLED 1 -#define MSR_KVM_PV_EOI_EN 0x4b564d04 -#define KVM_PV_EOI_BIT 0 -#define KVM_PV_EOI_MASK (0x1 << KVM_PV_EOI_BIT) -#define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK -#define KVM_PV_EOI_DISABLED 0x0 - -__attribute__ ((aligned(4))) -static volatile unsigned long kvm_exitless_eoi = KVM_PV_EOI_DISABLED; - -void kvm_pv_eoi() -{ - uint8_t reg; - asm("btr %2, %0; setc %1" : "+m"(kvm_exitless_eoi), "=rm"(reg) : "r"(0)); - if (reg) { - kprintf("avoided\n"); - return; - } - // fallback to normal x2APIC EOI - x2apic_send_eoi(); -} -void kvm_pv_eoi_init() -{ - union { - uint32_t msr[2]; - uint64_t whole; - } guest; - guest.whole = (uint64_t) &kvm_exitless_eoi; - guest.whole |= KVM_MSR_ENABLED; - x86::CPU::write_msr(MSR_KVM_PV_EOI_EN, guest.msr[0], guest.msr[1]); - // verify that the feature was enabled - uint64_t res = x86::CPU::read_msr(MSR_KVM_PV_EOI_EN); - if (res & 1) { - kprintf("* KVM paravirtual EOI enabled\n"); - // set new EOI handler - current_eoi_mechanism = kvm_pv_eoi; - } -} diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 03d0a13911..23d146d5e2 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -52,15 +52,11 @@ struct alignas(SMP_ALIGN) OS_CPU { }; static SMP_ARRAY os_per_cpu; -uint64_t OS::nanos_since_boot() noexcept { - return __arch_system_time(); -} - uint64_t OS::cycles_asleep() noexcept { return PER_CPU(os_per_cpu).cycles_hlt; } uint64_t OS::nanos_asleep() noexcept { - return PER_CPU(os_per_cpu).cycles_hlt / cpu_freq().count(); + return (PER_CPU(os_per_cpu).cycles_hlt * 1e6) / cpu_freq().count(); } __attribute__((noinline)) @@ -97,11 +93,6 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) MYINFO("Stack: %p", get_cpu_esp()); MYINFO("Boot magic: 0x%x, addr: 0x%x", boot_magic, boot_addr); - /// STATMAN /// - PROFILE("Statman"); - /// initialize on page 7, 3 pages in size - Statman::get().init(0x6000, 0x3000); - // Call global ctors PROFILE("Global constructors"); //__libc_init_array(); @@ -123,18 +114,24 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) // Give the rest of physical memory to heap OS::heap_max_ = OS::memory_end_; + /// STATMAN /// + PROFILE("Statman"); + /// initialize on page 9, 8 pages in size + Statman::get().init(0x8000, 0x8000); + PROFILE("Memory map"); // Assign memory ranges used by the kernel auto& memmap = memory_map(); MYINFO("Assigning fixed memory ranges (Memory map)"); - memmap.assign_range({0x6000, 0x8fff, "Statman", "Statistics"}); + memmap.assign_range({0x8000, 0xffff, "Statman", "Statistics"}); #if defined(ARCH_x86_64) - memmap.assign_range({0x100000, 0x8fffff, "Pagetables", "System page tables"}); - memmap.assign_range({0x900000, 0x9fffff, "Stack", "System main stack"}); + memmap.assign_range({0x1000, 0x6fff, "Pagetables", "System page tables"}); + memmap.assign_range({0x10000, 0x9d3ff, "Stack", "System main stack"}); #elif defined(ARCH_i686) - memmap.assign_range({0xA000, 0x9fbff, "Stack", "System main stack"}); + memmap.assign_range({0x10000, 0x9d3ff, "Stack", "System main stack"}); #endif + //memmap.assign_range({0x9d400, 0x9ffff, "Multiboot", "Multiboot reserved area"}); memmap.assign_range({(uintptr_t)&_LOAD_START_, (uintptr_t)&_end - 1, "ELF", "Your service binary including OS"}); @@ -154,7 +151,6 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) for (const auto &i : memmap) INFO2("* %s",i.second.to_string().c_str()); - PROFILE("Platform init"); extern void __platform_init(); __platform_init(); diff --git a/src/platform/x86_pc/platform.cpp b/src/platform/x86_pc/platform.cpp index 51e6ea5f81..ca001e5dee 100644 --- a/src/platform/x86_pc/platform.cpp +++ b/src/platform/x86_pc/platform.cpp @@ -86,7 +86,7 @@ void __platform_init() initialize_gdt_for_cpu(APIC::get().get_id()); #ifdef ARCH_x86_64 // setup Interrupt Stack Table - x86::ist_initialize_for_cpu(0, 0xA00000); + x86::ist_initialize_for_cpu(0, 0x9D3F0); #endif // IDT manager: Interrupt and exception handlers diff --git a/src/platform/x86_pc/smbios.cpp b/src/platform/x86_pc/smbios.cpp index f4f23621f5..b1458b63a9 100644 --- a/src/platform/x86_pc/smbios.cpp +++ b/src/platform/x86_pc/smbios.cpp @@ -20,6 +20,7 @@ #include #include #include +#include namespace x86 { @@ -78,6 +79,32 @@ namespace x86 } }; + struct PhysMemArray : public Header + { + struct bits_t + { + uint8_t location; + uint8_t use; + uint8_t error_corr_mtd; + uint32_t capacity32; + uint16_t mem_err_info_handle; + uint16_t num_slots; + uint64_t capacity64; + + } __attribute__((packed)); + + const bits_t& info() const noexcept { + return *(bits_t*) &data[0]; + } + + uintptr_t capacity() const noexcept { + const auto& inf = info(); + return (inf.capacity32 == 0x80000000) + ? inf.capacity64 : inf.capacity32 * 1024; + } + + }; + void SMBIOS::parse(const char* mem) { auto* table = (const EntryPoint*) mem; @@ -109,11 +136,24 @@ namespace x86 INFO2("System UUID: %s", sysinfo.uuid.c_str()); } break; + case 16: + { + const auto* array = (PhysMemArray*) hdr; + sysinfo.physical_memory = array->capacity(); + INFO2("Physical memory array with %lu MB capacity", + sysinfo.physical_memory / (1024*1024)); + } + break; } hdr = hdr->next(); if (hdr->type == 0) break; } + + // salvage operation for when no memory array found + if (sysinfo.physical_memory == 0) { + sysinfo.physical_memory = OS::memory_end()+1; + } } static uint8_t checksum(const char* addr, int length) diff --git a/src/platform/x86_pc/softreset.cpp b/src/platform/x86_pc/softreset.cpp index b72fec92b4..fb7b72c171 100644 --- a/src/platform/x86_pc/softreset.cpp +++ b/src/platform/x86_pc/softreset.cpp @@ -3,7 +3,7 @@ #include #define SOFT_RESET_MAGIC 0xFEE1DEAD -#define SOFT_RESET_LOCATION 0x7000 +#define SOFT_RESET_LOCATION 0x8200 namespace x86 { extern uint32_t apic_timer_get_ticks() noexcept; diff --git a/src/platform/x86_pc/start.asm b/src/platform/x86_pc/start.asm index e36b4f8303..313fbd238d 100644 --- a/src/platform/x86_pc/start.asm +++ b/src/platform/x86_pc/start.asm @@ -27,6 +27,10 @@ global __avx_enabled %define MB_MAGIC 0x1BADB002 %define MB_FLAGS 0x3 ;; ALIGN + MEMINFO +;; stack base address at EBDA border +;; NOTE: Multiboot can use 9d400 to 9ffff +%define STACK_LOCATION 0x9D3F0 + extern _MULTIBOOT_START_ extern _LOAD_START_ extern _LOAD_END_ @@ -76,9 +80,8 @@ rock_bottom: mov cx, 0x18 ;; GS segment mov gs, cx - ;; 32-bit stack base address at EBDA border - ;; NOTE: Multiboot can use 9fc00 to 9ffff - mov esp, 0x9FC00 + ;; 32-bit stack ptr + mov esp, STACK_LOCATION mov ebp, esp ;; enable SSE before we enter C/C++ land diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index fe521ce8ff..ecc5fd9266 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -63,10 +62,6 @@ uint64_t OS::nanos_asleep() noexcept { return os_cycles_hlt; } -uint64_t OS::nanos_since_boot() noexcept { - return solo5_clock_monotonic(); -} - void OS::default_stdout(const char* str, const size_t len) { solo5_console_write(str, len); @@ -141,10 +136,6 @@ void OS::start(char* _cmdline, uintptr_t mem_size) // timer stop function [] () {}); - // Some tests are asserting there is at least one timer that is always ON - // (the RTC calibration timer). Let's fake some timer so those tests pass. - Timers::oneshot(std::chrono::hours(1000000), [] (auto) {}); - Timers::ready(); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a7df102209..a353f25a3a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -184,10 +184,6 @@ set(OS_SOURCES ${SRC}/net/tcp/tcp.cpp ${SRC}/net/tcp/read_buffer.cpp ${SRC}/net/tcp/write_queue.cpp - ${SRC}/posix/fcntl.cpp - ${SRC}/posix/fd.cpp - ${SRC}/posix/sys/select.cpp - ${SRC}/posix/arpa/inet.cpp ${SRC}/util/async.cpp ${SRC}/util/logger.cpp ${SRC}/util/path_to_regex.cpp diff --git a/test/kernel/integration/LiveUpdate/service.cpp b/test/kernel/integration/LiveUpdate/service.cpp index 1a6e18ca76..b84914cc7c 100644 --- a/test/kernel/integration/LiveUpdate/service.cpp +++ b/test/kernel/integration/LiveUpdate/service.cpp @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include #include "liu.hpp" @@ -27,7 +27,7 @@ void Service::start() { auto func = begin_test_boot(); - if (liu::LiveUpdate::is_resumable() == false) + if (OS::is_live_updated() == false) { auto& inet = net::Super_stack::get(0); inet.network_config({10,0,0,49}, {255,255,255,0}, {10,0,0,1}); diff --git a/test/kernel/integration/LiveUpdate/test_boot.cpp b/test/kernel/integration/LiveUpdate/test_boot.cpp index 0193ee507f..12f6adb999 100644 --- a/test/kernel/integration/LiveUpdate/test_boot.cpp +++ b/test/kernel/integration/LiveUpdate/test_boot.cpp @@ -1,8 +1,10 @@ #include +#include #include +#include using namespace liu; -static std::vector timestamps; +static std::vector timestamps; static buffer_t bloberino; static void boot_save(Storage& storage, const buffer_t* blob) @@ -14,7 +16,7 @@ static void boot_save(Storage& storage, const buffer_t* blob) } static void boot_resume_all(Restore& thing) { - timestamps = thing.as_vector(); thing.go_next(); + timestamps = thing.as_vector(); thing.go_next(); // calculate time spent auto t1 = timestamps.back(); auto t2 = OS::nanos_since_boot(); @@ -30,6 +32,9 @@ LiveUpdate::storage_func begin_test_boot() { if (LiveUpdate::resume("test", boot_resume_all)) { + // OS must be able to tell it was live updated each time + assert(OS::is_live_updated()); + if (timestamps.size() >= 30) { // calculate median by sorting @@ -43,8 +48,13 @@ LiveUpdate::storage_func begin_test_boot() printf("%lld\n", stamp); } */ - printf("SUCCESS\n"); - OS::shutdown(); + printf("Verifying that timers are started...\n"); + + using namespace std::chrono; + Timers::oneshot(5ms,[] (int) { + printf("SUCCESS\n"); + }); + return nullptr; } else { // immediately liveupdate diff --git a/test/kernel/integration/modules/CMakeLists.txt b/test/kernel/integration/modules/CMakeLists.txt index 8c99d05032..123cc2b3b9 100644 --- a/test/kernel/integration/modules/CMakeLists.txt +++ b/test/kernel/integration/modules/CMakeLists.txt @@ -14,7 +14,6 @@ MESSAGE(STATUS "CMake root: " $ENV{INCLUDEOS_PREFIX}) set(SERVICE_NAME "Kernel modules test") set(BINARY "test_mods") -set(MAX_MEM 128) set(SOURCES service.cpp hotswap.cpp ) diff --git a/test/kernel/integration/modules/hotswap.cpp b/test/kernel/integration/modules/hotswap.cpp index 6f9e719274..3fa1610740 100644 --- a/test/kernel/integration/modules/hotswap.cpp +++ b/test/kernel/integration/modules/hotswap.cpp @@ -21,15 +21,11 @@ * function copied to an otherwise unused place in memory so that we can * overwrite the currently running binary with a new one. */ -#include -asm(".org 0x200000"); - -extern "C" void* __multiboot_magic; -extern "C" void* __multiboot_addr; +asm(".org 0x2000"); extern "C" __attribute__((noreturn)) -void hotswap(const char* base, int len, char* dest, void* start, - uintptr_t magic, uintptr_t bootinfo) +void hotswap(const char* base, int len, char* dest, + void* start, void* magic, void* bootinfo) { // Copy binary to its destination for (int i = 0; i < len; i++) diff --git a/test/kernel/integration/modules/mod2/CMakeLists.txt b/test/kernel/integration/modules/mod2/CMakeLists.txt index 166ccb4791..75c78594df 100644 --- a/test/kernel/integration/modules/mod2/CMakeLists.txt +++ b/test/kernel/integration/modules/mod2/CMakeLists.txt @@ -19,9 +19,6 @@ set(SERVICE_NAME "IncludeOS seed") # Name of your service binary set(BINARY "seed") -# Maximum memory can be hard-coded into the binary -set(MAX_MEM 128) - # Source files to be linked with OS library parts to form bootable image set(SOURCES service.cpp # ...add more here diff --git a/test/kernel/integration/modules/service.cpp b/test/kernel/integration/modules/service.cpp index a47989ff13..9588e639d2 100644 --- a/test/kernel/integration/modules/service.cpp +++ b/test/kernel/integration/modules/service.cpp @@ -43,15 +43,15 @@ void Service::start(const std::string& args) mod.mod_start, mod.mod_end, mod.mod_end - mod.mod_start); // Verify module cmdlines - Expects(std::string((char*)mods[0].cmdline) == "../mod1.json"); - Expects(std::string((char*)mods[1].cmdline) == "../seed loaded as module"); - Expects(std::string((char*)mods[2].cmdline) == "../mod3.json"); + Expects(std::string((char*) mods[0].cmdline) == "../mod1.json"); + Expects(std::string((char*) mods[1].cmdline) == "../seed loaded as module"); + Expects(std::string((char*) mods[2].cmdline) == "../mod3.json"); // verify content of text modules - Expects(std::string((char*)mods[0].mod_start) + Expects(std::string((char*) mods[0].mod_start) == "{\"module1\" : \"JSON data\" }\n"); - Expects(std::string((char*)mods[2].mod_start) + Expects(std::string((char*) mods[2].mod_start) == "{\"module3\" : \"More JSON data, for mod2 service\" }\n"); multiboot_module_t binary = mods[1]; @@ -60,10 +60,10 @@ void Service::start(const std::string& args) Elf_binary elf ({(char*)binary.mod_start, (int)(binary.mod_end - binary.mod_start)}); - void* hotswap_addr = (void*)0x100000; + void* hotswap_addr = (void*)0x2000; MYINFO("Moving hotswap function (now at %p)", &hotswap); - memcpy(hotswap_addr, (void*)&hotswap, 1024); + memcpy(hotswap_addr, (void*)&hotswap, 2048); extern uintptr_t __multiboot_magic; extern uintptr_t __multiboot_addr; @@ -72,9 +72,9 @@ void Service::start(const std::string& args) auto load_offs = elf.program_headers()[0].p_offset; char* base = (char*)binary.mod_start + load_offs; - int len = (int)(binary.mod_end - binary.mod_start); - char* dest = (char*)0xA00000; - void* start = (void*)elf.entry(); + int len = int(binary.mod_end - binary.mod_start); + char* dest = (char*) elf.program_headers()[0].p_paddr; + void* start = (void*) elf.entry(); SHA1 sha; sha.update(base, len); @@ -87,8 +87,7 @@ void Service::start(const std::string& args) MYINFO("Disabling interrupts and calling hotswap..."); asm("cli"); - ((decltype(&hotswap))hotswap_addr)(base, len, dest, start, __multiboot_magic, __multiboot_addr); - + ((decltype(&hotswap))hotswap_addr)(base, len, dest, start, 0, 0); + //__multiboot_magic, __multiboot_addr); panic("Should have jumped\n"); - } diff --git a/test/kernel/integration/timers/timers.cpp b/test/kernel/integration/timers/timers.cpp index 2fe0375f25..0ca2125868 100644 --- a/test/kernel/integration/timers/timers.cpp +++ b/test/kernel/integration/timers/timers.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include @@ -28,8 +28,9 @@ static int repeat2 = 0; void test_timers() { - INFO("Timers", "Testing one-shot timers"); - static int BASE_TIMERS; + INFO("Timers", "Testing kernel timers"); + // a calibration timer is active on bare metal and in emulated environments + static size_t BASE_TIMERS; BASE_TIMERS = Timers::active(); // 30 sec. - Test End diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index 5d1b150c15..caf8d30cb8 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -65,9 +65,6 @@ void OS::start(unsigned, unsigned) {} void OS::default_stdout(const char*, size_t) {} void OS::event_loop() {} void OS::block() {} -uint64_t OS::nanos_since_boot() noexcept { - return 0; -} void OS::resume_softreset(intptr_t) {} bool OS::is_softreset_magic(uint32_t) { return true; @@ -91,6 +88,20 @@ extern "C" { return 0xdeadbeef; } + void kprintf(const char* format, ...){ + va_list args; + va_start (args, format); + vprintf (format, args); + va_end (args); + } + + void kprint(const char* str){ + printf(str); + } + + void* heap_end; + void* heap_begin; + /// C ABI /// void _init_c_runtime() {} void _init_bss() {} diff --git a/vmrunner/vm.schema.json b/vmrunner/vm.schema.json index c2dff759ed..705317567d 100644 --- a/vmrunner/vm.schema.json +++ b/vmrunner/vm.schema.json @@ -99,6 +99,11 @@ "vga" : { "description" : "Enable VGA screen", "enum" : ["std", "cirrus", "vmware", "qxl", "xenfb", "tcx", "cg3", "virtio", "none"] + }, + + "vfio" : { + "description" : "VFIO PCI-passthrough on device", + "type" : "string" } } diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 288852a33e..38ba61eecb 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -383,13 +383,16 @@ def boot(self, multiboot, debug = False, kernel_args = "", image_name = None): if "vga" in self._config: vga_arg = ["-vga", str(self._config["vga"])] + pci_arg = [] + if "vfio" in self._config: + pci_arg = ["-device", "vfio-pci,host=" + self._config["vfio"]] + # TODO: sudo is only required for tap networking and kvm. Check for those. command = ["sudo", "--preserve-env", "qemu-system-x86_64"] if self._kvm_present: command.extend(["--enable-kvm"]) command += kernel_args - - command += disk_args + debug_args + net_args + mem_arg + vga_arg + mod_args + command += disk_args + debug_args +net_args + mem_arg + vga_arg + pci_arg + mod_args #command_str = " ".join(command) #command_str.encode('ascii','ignore') From 8f59fd13527f93647846e596094fb7c9c71e2a5d Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 6 Feb 2018 13:08:35 +0100 Subject: [PATCH 042/723] vmrunner: add back optional exit on panic --- vmrunner/vmrunner.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 38ba61eecb..628d45f1ae 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -37,6 +37,7 @@ # (One default vm added at the end) vms = [] +panic_signature = "\\x15\\x07\\t\*\*\*\* PANIC \*\*\*\*" nametag = "" INFO = color.INFO(nametag) VERB = bool(os.environ["VERBOSE"]) if "VERBOSE" in os.environ else False @@ -392,7 +393,7 @@ def boot(self, multiboot, debug = False, kernel_args = "", image_name = None): if self._kvm_present: command.extend(["--enable-kvm"]) command += kernel_args - command += disk_args + debug_args +net_args + mem_arg + vga_arg + pci_arg + mod_args + command += disk_args + debug_args + net_args + mem_arg + vga_arg + pci_arg + mod_args #command_str = " ".join(command) #command_str.encode('ascii','ignore') @@ -485,7 +486,7 @@ def __init__(self, config = None, hyper = qemu): self._on_panic = self.panic self._on_timeout = self.timeout self._on_output = { - "\\x15\\x07\\t\*\*\*\* PANIC \*\*\*\*" : self._on_panic, + panic_signature : self._on_panic, "SUCCESS" : self._on_success } # Initialize hypervisor with config @@ -584,8 +585,10 @@ def on_success(self, callback, do_exit = True): else: self._on_output["SUCCESS"] = callback return self - def on_panic(self, callback): - self._on_output["PANIC"] = lambda(line) : [callback(line), self._on_panic(line)] + def on_panic(self, callback, do_exit = True): + if do_exit: + self._on_output[panic_signature] = lambda(line) : [callback(line), self._on_panic(line)] + else: self._on_output[panic_signature] = callback return self def on_timeout(self, callback): From d427970cf154259913460c073714d13cadee94b4 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 8 Feb 2018 12:30:25 +0100 Subject: [PATCH 043/723] Fix merge conflicts --- examples/demo_service/CMakeLists.txt | 8 ++-- src/arch/x86_64/linker.ld | 13 ++---- src/musl/CMakeLists.txt | 2 +- src/musl/brk.cpp | 60 +++++++++++++++++++--------- src/musl/exit.cpp | 2 + src/platform/x86_pc/kernel_start.cpp | 18 ++------- src/virtio/virtio_queue.cpp | 3 ++ 7 files changed, 60 insertions(+), 46 deletions(-) diff --git a/examples/demo_service/CMakeLists.txt b/examples/demo_service/CMakeLists.txt index 31d8060e8c..cdde788e5f 100644 --- a/examples/demo_service/CMakeLists.txt +++ b/examples/demo_service/CMakeLists.txt @@ -29,10 +29,12 @@ if ("$ENV{PLATFORM}" STREQUAL "x86_solo5") ) else() set(DRIVERS - virtionet # Virtio networking - # virtioblk # Virtio block device + virtionet # Virtio networking + # virtioblk # Virtio block device - # Use "boot --drivers ." to see other drivers + # Use "boot --drivers ." to see other drivers + # virtioblock # Virtio block device + # ... Others from src/drivers ) endif() diff --git a/src/arch/x86_64/linker.ld b/src/arch/x86_64/linker.ld index 049e1c451d..eba07c0a5e 100644 --- a/src/arch/x86_64/linker.ld +++ b/src/arch/x86_64/linker.ld @@ -21,13 +21,15 @@ ENTRY(_start) SECTIONS { - PROVIDE (__executable_start = SEGMENT_START("text-segment", 0x100000)); - . = SEGMENT_START("text-segment", 0x100000) + SIZEOF_HEADERS; + + PROVIDE (__executable_start = SEGMENT_START("text-segment", 0x800000)); /* For convenience w. multiboot */ PROVIDE ( _ELF_START_ = __executable_start); PROVIDE ( _LOAD_START_ = _ELF_START_ ); + . = SEGMENT_START("text-segment", 0x800000) + SIZEOF_HEADERS; + .multiboot : { PROVIDE(_MULTIBOOT_START_ = . ); *(.multiboot) @@ -143,13 +145,6 @@ SECTIONS _erodata = .; } - .config ALIGN(0x8) : { - _CONFIG_JSON_START_ = .; - KEEP(*(.config)) - _CONFIG_JSON_END_ = .; - BYTE(0); - } - .eh_frame : { PROVIDE (__eh_frame_start = .); diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index bfa96b4d9f..5fc6514805 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -7,7 +7,7 @@ set(MUSL_OBJECTS nanosleep.cpp open.cpp clock_gettime.cpp gettimeofday.cpp poll.cpp mmap2.cpp exit_group.cpp exit.cpp close.cpp set_tid_address.cpp pipe.cpp read.cpp readv.cpp getpid.cpp mknod.cpp stat.cpp sync.cpp msync.cpp - mincore.cpp syscall_n.cpp sigmask.cpp + mincore.cpp syscall_n.cpp sigmask.cpp gettid.cpp tkill.cpp ) add_library(musl_syscalls STATIC ${MUSL_OBJECTS}) diff --git a/src/musl/brk.cpp b/src/musl/brk.cpp index 761223fec5..91fe708697 100644 --- a/src/musl/brk.cpp +++ b/src/musl/brk.cpp @@ -2,36 +2,58 @@ #include #include #include +#include -uintptr_t heap_begin; -uintptr_t heap_end; -uintptr_t max_brk; +__attribute__((weak)) +uintptr_t __brk_max = 0x10000; + +uintptr_t heap_begin = 0; + +uintptr_t brk_end; +uintptr_t brk_init = 0; + +extern void init_mmap(uintptr_t mmap_begin); + +extern "C" +void _init_heap(uintptr_t free_mem_begin) +{ + #define HEAP_ALIGNMENT 4095 + // NOTE: Initialize the heap before exceptions + // cache-align heap, because its not aligned + heap_begin = free_mem_begin + HEAP_ALIGNMENT; + heap_begin = ((uintptr_t)heap_begin & ~HEAP_ALIGNMENT); + brk_end = heap_begin; + brk_init = brk_end; + kprintf("* Brk initialized. Begin: 0x%lx, end 0x%lx\n", + heap_begin, brk_end); + + init_mmap(heap_begin + __brk_max); + +} -#warning stub extern "C" uintptr_t syscall_SYS_brk(void* addr) { STRACE("syscall brk. Heap begin=0x%lx, OS::heap_max=0x%lx, addr=%p" "\n\t requests %i bytes \n", - heap_begin, OS::heap_max(), addr, (uintptr_t)addr - heap_end); - - // Keep track of total used heap - if (max_brk == 0) - max_brk = heap_end; + heap_begin, OS::heap_max(), addr, (uintptr_t)addr - brk_end); if (addr == nullptr - or (uintptr_t)addr > OS::heap_max() - or (uintptr_t)addr < heap_begin) - return heap_end; + or (uintptr_t)addr > heap_begin + __brk_max + or (uintptr_t)addr < heap_begin) { + kprintf("\tBrk failed\n"); + return brk_end; + } - heap_end = (uintptr_t)addr; + brk_end = (uintptr_t)addr; - if (heap_end > max_brk) { - kprintf("\t Initializing %i b\n", heap_end - max_brk); - memset((void*)max_brk, 0, heap_end - max_brk); - max_brk = heap_end; + if (brk_end > brk_init) { + kprintf("\t Initializing %i b\n", brk_end - brk_init); + memset((void*)brk_init, 0, brk_end - brk_init); + brk_init = brk_end; } - kprintf("\t Done, returning 0x%lx\n", heap_end); - return heap_end; + + kprintf("\t Done, returning 0x%lx\n", brk_end); + return brk_end; } diff --git a/src/musl/exit.cpp b/src/musl/exit.cpp index 812c2ffb5c..6449602ab6 100644 --- a/src/musl/exit.cpp +++ b/src/musl/exit.cpp @@ -1,7 +1,9 @@ #include "common.hpp" +#include extern "C" long syscall_SYS_exit() { STUB("exit"); + panic("Exiting\n"); return 0; } diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 3fc8f6960b..f258dcaaf4 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -32,10 +32,9 @@ extern "C" { void __test_syscall(void* p); uintptr_t _multiboot_free_begin(uintptr_t boot_addr); uintptr_t _move_symbols(uintptr_t loc); + void _init_heap(uintptr_t free_mem_begin); } -extern uintptr_t heap_begin; -extern uintptr_t heap_end; extern char _ELF_START_; extern char _ELF_END_; extern char _INIT_START_; @@ -43,17 +42,6 @@ extern char _FINI_START_; thread_local int __tl1__ = 42; -void _init_heap(uintptr_t free_mem_begin) -{ - #define HEAP_ALIGNMENT 4095 - // NOTE: Initialize the heap before exceptions - // cache-align heap, because its not aligned - heap_begin = free_mem_begin + HEAP_ALIGNMENT; - heap_begin = ((uintptr_t)heap_begin & ~HEAP_ALIGNMENT); - heap_end = heap_begin; - kprintf("* Heap initialized. Begin: 0x%lx, end 0x%lx\n", - heap_begin, heap_end); -} void _init_bss() { @@ -175,8 +163,10 @@ void kernel_start(uintptr_t magic, uintptr_t addr) kprintf("* Free mem begin: %p \n", free_mem_begin); } + kprintf("* Moving symbols. \n"); // Preserve symbols from the ELF binary - //free_mem_begin += _move_symbols(free_mem_begin); + free_mem_begin += _move_symbols(free_mem_begin); + kprintf("* Free mem moved to: %p \n", free_mem_begin); kprintf("* Grub magic: 0x%lx, grub info @ 0x%lx\n", __grub_magic, __grub_addr); diff --git a/src/virtio/virtio_queue.cpp b/src/virtio/virtio_queue.cpp index 1a0e8b65ba..35f9f580bd 100644 --- a/src/virtio/virtio_queue.cpp +++ b/src/virtio/virtio_queue.cpp @@ -67,6 +67,9 @@ Virtio::Queue::Queue(const std::string& name, size_t total_bytes = virtq_size(size); // Allocate page-aligned size and clear it void* buffer = memalign(PAGE_SIZE, total_bytes); + if (! buffer) + panic("Virtio queue could not allocate aligned queue area"); + memset(buffer, 0, total_bytes); debug(">>> Virtio Queue %s of size %i (%u bytes) initializing \n", From 520b0c5807e1410f7adaa66f1433adc0ac70bb90 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 8 Feb 2018 13:55:47 +0100 Subject: [PATCH 044/723] musl: basic allocator for mmap --- api/util/alloc_lstack.hpp | 129 ++++++++++++++++++++++++++++++++++++++ src/musl/brk.cpp | 2 +- src/musl/mmap.cpp | 60 +++++++++++++++++- src/musl/munmap.cpp | 7 ++- 4 files changed, 193 insertions(+), 5 deletions(-) create mode 100644 api/util/alloc_lstack.hpp diff --git a/api/util/alloc_lstack.hpp b/api/util/alloc_lstack.hpp new file mode 100644 index 0000000000..9df14ca1e9 --- /dev/null +++ b/api/util/alloc_lstack.hpp @@ -0,0 +1,129 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef UTIL_MINIALLOC_HPP +#define UTIL_MINIALLOC_HPP + +#include + +namespace util { +namespace alloc { + +/** + * Lazy stack of memory blocks. + * A stack-like data structure for arbitrarily sized blocks. + * - Uses free blocks as storage for the nodes (no memory overhead). This also + * implies that the first part of each free block must be present in memory. + * - Lazily chops smaller blocks off of larger blocks as requested. + * - Constant time first time allocation of any sized block. + * - LIFO stack behavior for reallocating uniformly sized blocks (pop) + * - Constant time deallocation (push). + * - Linear search, first fit, for reallocatiing arbitrarily sized blocks. + * The first block larger than the requested size will be chopped to match, + * which implies that the size of the blocks will converge on Min and the + * number of blocks will converge on total size / Min for arbitrarily sized + * allocations. + * + * The allocator *can* be used as a general purpose allocator, but will be slow + * for that purpose. E.g. after n arbitrarily sized deallocations the complexity + * of allocating a block of size k will be O(n). + * + * NOTE: The allocator will not search for duplicates on deallocation, + * e.g. won't prevent double free, so that has to be done elsewhere. + **/ +template +class Lstack { +public: + + struct Chunk { + Chunk* next; + size_t size; + Chunk(Chunk* n, size_t s) + : next(n), size(s) + { + Expects(s >= Min); + } + }; + + static constexpr int align = Min; + + void* allocate(ssize_t size){ + Expects((size & (Min - 1)) == 0); + + Chunk* next = front_; + Chunk** prev = &front_; + do { + + if (UNLIKELY(next == nullptr)) + return nullptr; + + // Cleanly take out chunk as-is + if (next->size == size) + { + *prev = next->next; + return next; + } + + // Clip off size from chunk + if ( next->size > size) + { + if (next->size - size <= 0 and next->next == nullptr) + *prev = nullptr; + else + *prev = new_chunk((char*)next + size, next->next, next->size - size); + return next; + } + + prev = &(next->next); + } while ((next = next->next)); + + // No suitable chunk + return nullptr; + } + + void deallocate (void* ptr, size_t size){ + push(ptr, size); + } + + void donate(void* ptr, ssize_t size){ + push(ptr, size); + } + + void push(void* ptr, ssize_t size){ + Expects((size & (align - 1)) == 0); + auto* old_front = front_; + front_ = (Chunk*)ptr; + new_chunk(front_, old_front, size); + }; + + const Chunk* begin() const { + return front_; + } + + bool empty() const noexcept { return front_ == nullptr || front_->size == 0; } + +private: + Chunk* new_chunk(void* addr_begin, Chunk* next, size_t sz){ + Expects((sz & align - 1) == 0); + Expects(((uintptr_t)addr_begin & align - 1) == 0); + return new ((Chunk*)addr_begin) Chunk(next, sz); + } + Chunk* front_ = nullptr; +}; + +} // namespace util +} // namespace alloc +#endif diff --git a/src/musl/brk.cpp b/src/musl/brk.cpp index 91fe708697..7f04b29575 100644 --- a/src/musl/brk.cpp +++ b/src/musl/brk.cpp @@ -5,7 +5,7 @@ #include __attribute__((weak)) -uintptr_t __brk_max = 0x10000; +uintptr_t __brk_max = 0x100000; uintptr_t heap_begin = 0; diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 511b92aefd..35c320f098 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -1,19 +1,73 @@ #include "common.hpp" #include +#include +#include +#include +#include extern uintptr_t heap_begin; +extern uintptr_t heap_end = 0; static uintptr_t current_pos = 0; + +using Alloc = util::alloc::Lstack<4096>; +static Alloc alloc; + +void init_mmap(uintptr_t addr_begin){ + if (alloc.empty()){ + printf("Alloc not empty: %s\n ", alloc.begin() == nullptr ? "nullptr" : "size not 0"); + } + Expects(alloc.empty()); + auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); + alloc.donate((void*)aligned_begin, (OS::heap_max() - aligned_begin) & ~(Alloc::align - 1)); + +} + +extern "C" +void* __kalloc(size_t size){ + return alloc.allocate(size); +} + +extern "C" +void __kfree (void* ptr, size_t size){ + alloc.deallocate(ptr, size); +} + extern "C" void* syscall_SYS_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) { - uintptr_t res = heap_begin + current_pos; - current_pos += length; + + STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d \n", + addr, length, prot, flags, fd, offset); + + // TODO: Mapping to file descriptor + if (fd > 0) { + assert(false && "Mapping to file descriptor not yet implemented"); + } + + // TODO: mapping virtual address + if (addr) { + errno = ENODEV; + return MAP_FAILED; + } + + void* res = __kalloc(length); STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d res=%p\n", addr, length, prot, flags, fd, offset, res); - return (void*)res; + + return res; } + +/** + The mmap2() system call provides the same interface as mmap(2), + except that the final argument specifies the offset into the file in + 4096-byte units (instead of bytes, as is done by mmap(2)). This + enables applications that use a 32-bit off_t to map large files (up + to 2^44 bytes). + + http://man7.org/linux/man-pages/man2/mmap2.2.html +**/ extern "C" void* syscall_SYS_mmap2(void *addr, size_t length, int prot, int flags, int fd, off_t offset) { diff --git a/src/musl/munmap.cpp b/src/musl/munmap.cpp index 837a26be3c..a8ff4e172a 100644 --- a/src/musl/munmap.cpp +++ b/src/musl/munmap.cpp @@ -1,7 +1,12 @@ #include "common.hpp" extern "C" -long syscall_SYS_munmap() { +void __kfree(void* addr, size_t length); + +extern "C" +int syscall_SYS_munmap(void *addr, size_t length) +{ STUB("munmap"); + __kfree(addr, length); return 0; } From dfbda109756638aa9fec6cb54a1710c1076bc980 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 8 Feb 2018 13:59:39 +0100 Subject: [PATCH 045/723] musl: integrate with new bundle (libc++ 5 threaded), more fixes --- cmake/cross_compiled_libraries.cmake | 4 ++-- src/CMakeLists.txt | 2 +- src/arch/x86_64/arch_start.asm | 5 ++++- src/crt/c_abi.c | 18 +----------------- src/include/kprint | 6 +++++- src/musl/gettid.cpp | 11 +++++++++++ src/musl/tkill.cpp | 12 ++++++++++++ 7 files changed, 36 insertions(+), 22 deletions(-) create mode 100644 src/musl/gettid.cpp create mode 100644 src/musl/tkill.cpp diff --git a/cmake/cross_compiled_libraries.cmake b/cmake/cross_compiled_libraries.cmake index 081236e3b2..65988020c6 100644 --- a/cmake/cross_compiled_libraries.cmake +++ b/cmake/cross_compiled_libraries.cmake @@ -18,8 +18,8 @@ else(BUNDLE_LOC) include(ExternalProject) ExternalProject_Add(PrecompiledLibraries PREFIX precompiled - URL https://github.com/hioa-cs/IncludeOS/releases/download/v0.12.0-rc.2/IncludeOS_dependencies_v0-12-0_musl_libunwind.tar.gz - URL_HASH SHA1=4c275a797c1417ef2a2b5cbedc0f947dd7b31f7d + URL https://github.com/hioa-cs/IncludeOS/releases/download/v0.12.0-rc.2/IncludeOS_dependencies_v0-12-0_musl_libunwind_threaded.tar.gz + URL_HASH SHA1=4ab139536295321866ebe4136de25879450eb58b CONFIGURE_COMMAND "" BUILD_COMMAND "" UPDATE_COMMAND "" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index da63580302..976d7754d6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -39,7 +39,7 @@ set(OS_OBJECTS util/syslog_facility.cpp util/syslogd.cpp util/uri.cpp util/percent_encoding.cpp util/tar.cpp util/path_to_regex.cpp util/config.cpp util/autoconf.cpp crt/ctype_b_loc.c crt/ctype_tolower_loc.c crt/string.c - crt/quick_exit.cpp crt/cxx_abi.cpp + crt/quick_exit.cpp crt/cxx_abi.cpp crt/c_abi.c hw/pci_device.cpp hw/ps2.cpp hw/serial.cpp hw/vga_gfx.cpp hw/msi.cpp hw/pci_msi.cpp virtio/virtio.cpp virtio/virtio_queue.cpp net/ethernet/ethernet.cpp net/ethernet/ethernet_8021q.cpp diff --git a/src/arch/x86_64/arch_start.asm b/src/arch/x86_64/arch_start.asm index d1b6b9cbad..14b2f85527 100644 --- a/src/arch/x86_64/arch_start.asm +++ b/src/arch/x86_64/arch_start.asm @@ -44,10 +44,13 @@ extern __multiboot_addr ;; Extended Feature Enable Register (MSR) %define IA32_EFER_MSR 0xC0000080 + ;; EFER Longmode bit %define LONGMODE_ENABLE 0x100 ;; EFER Execute Disable bit %define NX_ENABLE 0x800 +;; EFER Syscall enable bit +%define SYSCALL_ENABLE 0x1 [BITS 32] @@ -105,7 +108,7 @@ __arch_start: ;; enable long mode mov ecx, IA32_EFER_MSR rdmsr - or eax, (LONGMODE_ENABLE | NX_ENABLE) + or eax, (LONGMODE_ENABLE | NX_ENABLE | SYSCALL_ENABLE) wrmsr ;; enable paging diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index a55079330a..f3c1df8673 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -34,13 +34,6 @@ void* __dso_handle; const uintptr_t __stack_chk_guard = (uintptr_t) _STACK_GUARD_VALUE_; extern void panic(const char* why) __attribute__((noreturn)); -void _init_bss() -{ - /// Initialize .bss section - extern char _BSS_START_, _BSS_END_; - __builtin_memset(&_BSS_START_, 0, &_BSS_END_ - &_BSS_START_); -} - extern void (*__preinit_array_start [])(); extern void (*__preinit_array_end [])(); extern void (*__init_array_start [])(); @@ -61,16 +54,6 @@ void __libc_init_array() { __init_array_start[i](); } -void _init_heap(uintptr_t free_mem_begin) -{ - // NOTE: Initialize the heap before exceptions - // cache-align heap, because its not aligned - heap_begin = (void*) free_mem_begin + HEAP_ALIGNMENT; - heap_begin = (void*) ((uintptr_t)heap_begin & ~HEAP_ALIGNMENT); - // heap end tracking, used with sbrk - heap_end = heap_begin; -} - uint32_t _move_symbols(void* sym_loc) { extern char _ELF_SYM_START_; @@ -110,6 +93,7 @@ void _init_c_runtime() kprintf("Sanity checks OK \n"); } + // stack-protector __attribute__((noreturn)) void __stack_chk_fail(void) diff --git a/src/include/kprint b/src/include/kprint index f00ac745b4..3eceeab1b8 100644 --- a/src/include/kprint +++ b/src/include/kprint @@ -39,7 +39,11 @@ extern void __serial_print(const char* str, size_t len); * The earliest possible print function (requires no heap, global ctors etc.) **/ __attribute__ ((format (printf, 1, 2))) -extern "C" void kprintf(const char* format, ...); + +#ifdef __cplusplus +extern "C" +#endif +void kprintf(const char* format, ...); extern void kprint(const char*); #ifdef __cplusplus diff --git a/src/musl/gettid.cpp b/src/musl/gettid.cpp new file mode 100644 index 0000000000..63129ba9c3 --- /dev/null +++ b/src/musl/gettid.cpp @@ -0,0 +1,11 @@ +#include "common.hpp" + +extern "C" +long syscall_SYS_gettid() { + STRACE("gettid"); + #ifdef INCLUDEOS_SINGLE_THREADED + return 0; + #else + #error "gettid not implemented for threaded IncludeOS" + #endif +} diff --git a/src/musl/tkill.cpp b/src/musl/tkill.cpp new file mode 100644 index 0000000000..39c8990419 --- /dev/null +++ b/src/musl/tkill.cpp @@ -0,0 +1,12 @@ +#include "common.hpp" +#include + +extern "C" +int syscall_SYS_tkill(int tid, int sig) { + STRACE("tkill(%i, %i)", tid, sig); + #ifdef INCLUDEOS_SINGLE_THREADED + exit(sig); + #else + #error "tkill not implemented for threaded IncludeOS" + #endif +} From 6abf0a09b548e4ae72778d8fc7cafe246e2b6fd1 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 8 Feb 2018 14:00:24 +0100 Subject: [PATCH 046/723] test: lstack allocator unit test --- test/CMakeLists.txt | 3 + test/lest_util/os_mock.cpp | 16 +++- test/util/unit/lstack.cpp | 167 +++++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 test/util/unit/lstack.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5bb98d2d18..0efec45e14 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -124,6 +124,7 @@ set(TEST_SOURCES ${TEST}/util/unit/syslog_facility_test.cpp ${TEST}/util/unit/uri_test.cpp ${TEST}/util/unit/bitops.cpp + ${TEST}/util/unit/lstack.cpp ) set(OS_SOURCES @@ -199,6 +200,8 @@ set(OS_SOURCES ${SRC}/util/tar.cpp ${SRC}/util/uri.cpp ${SRC}/virtio/virtio_queue.cpp + ${SRC}/musl/mmap.cpp + ${SRC}/musl/brk.cpp ) set(MOD_OBJECTS diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index 879a649bae..911b07c76d 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -61,6 +61,21 @@ Timers::id_t Timers::periodic(duration_t, duration_t, handler_t) { const char* service_binary_name__ = "Service binary name"; const char* service_name__ = "Service name"; +extern "C" +void kprintf(char* format, ...) +{ + va_list args; + va_start(args, format); + vprintf(format, args); + va_end(args); +} + +extern "C" +void kprint(char* str) +{ +printf("%s", str); +} + #include void OS::start(unsigned, unsigned) {} void OS::default_stdout(const char*, size_t) {} @@ -110,7 +125,6 @@ extern "C" { /// C ABI /// void _init_c_runtime() {} void _init_bss() {} - void _init_heap(uintptr_t) {} #ifdef __MACH__ void _init() {} diff --git a/test/util/unit/lstack.cpp b/test/util/unit/lstack.cpp new file mode 100644 index 0000000000..8fbae527dd --- /dev/null +++ b/test/util/unit/lstack.cpp @@ -0,0 +1,167 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +using Lstack = util::alloc::Lstack<>; +using Chunk = Lstack::Chunk; + +void print_summary(const Chunk* begin) +{ + #ifdef NO_INFO + return; + #else + const Chunk* ptr = begin; + if (ptr == nullptr) { + printf("[0]\n"); + return; + } + + Expects(ptr->size != 0); + do { + Expects(ptr != nullptr); + printf("["); + for (int i = 0; i < ptr->size / 4096; i++) + { + printf("#"); + } + + if (ptr->next == nullptr) { + printf("]->0"); + break; + } else { + printf("]->"); + } + } while((ptr = ptr->next)); + printf("\n"); + #endif +} + +CASE("Using lstack") +{ + + Lstack heap; + EXPECT(heap.allocate(rand() & ~4095) == nullptr); + + auto poolsize = 0x100000; + auto blocksize = 0x1000; + + char* pool = (char*)memalign(blocksize, poolsize); + void* pool_end = pool + poolsize; + heap.donate(pool, poolsize); + + auto* page = heap.allocate(blocksize); + EXPECT(page == pool); + EXPECT(((Chunk*)page)->size == poolsize); + EXPECT(((Chunk*)page)->next == nullptr); + + print_summary(heap.begin()); + + int i = 0; + for (; i < poolsize - blocksize; i += blocksize) + { + auto* p = heap.allocate(blocksize); + EXPECT(p == (uint8_t*)pool + blocksize + i); + } + + print_summary(heap.begin()); + + EXPECT(heap.allocate(blocksize) == nullptr); + EXPECT(heap.begin() == nullptr); + + char* chunk1 = pool + 0x1000; + heap.deallocate(chunk1, 0x2000); + EXPECT((char*)heap.begin() == chunk1); + EXPECT((char*)heap.begin()->next == nullptr); + + print_summary(heap.begin()); + + auto* chunk2 = pool + 0x5000; + heap.deallocate(chunk2, 0x4000); + EXPECT((char*)heap.begin() == chunk2); + EXPECT((char*)heap.begin()->next == chunk1); + EXPECT((char*)heap.begin()->next->next == nullptr); + + print_summary(heap.begin()); + EXPECT(heap.allocate(0x4000) == chunk2); + print_summary(heap.begin()); + EXPECT(heap.allocate(0x1000) == chunk1); + EXPECT(heap.allocate(0x1000) == chunk1 + 0x1000); + EXPECT(heap.begin() == nullptr); + + // Free some small chunks in random order + void* prev = nullptr; + std::vector rands; + auto size = blocksize * 2; + auto count = poolsize / size; + auto index = 0; + + for (i = 0; i < 5; i++) + { + + do { + index = rand() % (count - 1); + } while (std::find(rands.begin(), rands.end(), index) != rands.end()); + + rands.push_back(index); + + auto* frag = &(pool)[index * size]; + EXPECT((frag >= pool && frag <= pool + poolsize - size)); + EXPECT(((uintptr_t)frag & blocksize - 1 ) == 0); + heap.donate(frag, size); + EXPECT((void*)heap.begin() == frag); + prev = frag; + print_summary(heap.begin()); + } + + print_summary(heap.begin()); + int blockcount = 0; + int total_size = 0; + auto* block = heap.begin(); + do { + EXPECT(block != nullptr); + EXPECT(((void*)block >= pool && (void*)block <= pool + poolsize)); + blockcount++; + total_size += block->size; + EXPECT(block->size == blocksize * 2); + } while ((block = block->next)); + + EXPECT(block == nullptr); + EXPECT(blockcount == 5); + EXPECT(total_size == 10 * blocksize); + + + // Fragment the first chunk + chunk1 = (char*)heap.allocate(0x1000); + EXPECT(chunk1 == pool + (rands.back() * size)); + print_summary(heap.begin()); + heap.deallocate(chunk1, 0x1000); + print_summary(heap.begin()); + EXPECT(heap.begin() == (Chunk*)chunk1); + EXPECT(heap.begin()->next->size == 0x1000); + EXPECT(heap.begin()->next->next->size == 0x2000); + + auto* first_2k = heap.begin()->next->next; + chunk2 = (char*)heap.allocate(0x2000); + print_summary(heap.begin()); + EXPECT((Chunk*)chunk2 == first_2k); + + free(pool); + + +} From d78dc407f890279d90f90e5fd780a4ce7bd0ef96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 9 Feb 2018 10:07:28 +0100 Subject: [PATCH 047/723] tcp: Changes to aligned_option in packet --- api/net/tcp/options.hpp | 8 +++++++- api/net/tcp/packet.hpp | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/api/net/tcp/options.hpp b/api/net/tcp/options.hpp index a6e6203202..4ffbb3e5e6 100644 --- a/api/net/tcp/options.hpp +++ b/api/net/tcp/options.hpp @@ -132,6 +132,9 @@ struct Option { : ts{val, echo} {} + uint8_t size() const noexcept + { return sizeof(padding) + sizeof(ts); } + } __attribute__((packed)); /** @@ -155,7 +158,7 @@ struct Option { template opt_sack(const Collection& blocks) - : length{static_cast(2 + + : length{static_cast(sizeof(kind) + sizeof(length) + (blocks.size() * sizeof(typename Collection::value_type)))} { using T = typename Collection::value_type; @@ -174,6 +177,9 @@ struct Option { : sack{blocks} {} + uint8_t size() const noexcept + { return sizeof(padding) + sack.length; } + } __attribute__((packed)); }; // < struct Option diff --git a/api/net/tcp/packet.hpp b/api/net/tcp/packet.hpp index 3472478927..37f867af78 100644 --- a/api/net/tcp/packet.hpp +++ b/api/net/tcp/packet.hpp @@ -257,15 +257,15 @@ class Packet : public PacketIP4 { template inline void add_tcp_option_aligned(Args&&... args) { // to avoid headache, options need to be added BEFORE any data. - Expects(!has_tcp_data() and sizeof(T) % 4 == 0); + Expects(!has_tcp_data()); // option address auto* addr = tcp_options()+tcp_options_length(); // emplace the option - new (addr) T(args...); + auto& opt = *(new (addr) T(args...)); // update offset - auto newoffset = offset() + (sizeof(T) / 4); + auto newoffset = offset() + round_up(opt.size(), 4); set_offset(newoffset); if (UNLIKELY(newoffset > 0xF)) { From 5e9b1dfd77e96b02f57d3485036aa52b566e39e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 9 Feb 2018 10:09:11 +0100 Subject: [PATCH 048/723] tcp: Changes to read_request (fits and buf rdy algo) --- api/net/tcp/read_request.hpp | 6 ++- src/net/tcp/read_request.cpp | 74 +++++++++++++++++++++++++++--------- 2 files changed, 62 insertions(+), 18 deletions(-) diff --git a/api/net/tcp/read_request.hpp b/api/net/tcp/read_request.hpp index a5c69eb4c8..1df547afcf 100644 --- a/api/net/tcp/read_request.hpp +++ b/api/net/tcp/read_request.hpp @@ -29,6 +29,7 @@ namespace tcp { class Read_request { public: using Buffer_ptr = std::unique_ptr; + using Buffer_queue = std::deque; using ReadCallback = delegate; static constexpr size_t buffer_limit = 2; ReadCallback callback; @@ -51,8 +52,11 @@ class Read_request { Read_buffer& front() { return *buffers.front(); } + const Buffer_queue& queue() const + { return buffers; } + private: - std::deque buffers; + Buffer_queue buffers; Read_buffer* get_buffer(const seq_t seq); diff --git a/src/net/tcp/read_request.cpp b/src/net/tcp/read_request.cpp index ba1bcfd362..1dd02715e3 100644 --- a/src/net/tcp/read_request.cpp +++ b/src/net/tcp/read_request.cpp @@ -36,8 +36,10 @@ namespace tcp { { auto* buf = get_buffer(seq); - if(UNLIKELY(buf == nullptr)) + if(UNLIKELY(buf == nullptr)) { + //printf("no buffer found\n"); break; + } //printf("got buffer start=%u end=%u missing=%lu\n", // buf->start_seq(), buf->end_seq(), buf->missing()); @@ -52,12 +54,20 @@ namespace tcp { // (and also being the one in the front) while(buf->is_ready() and buf == buffers.front().get()) { + const auto rem = buf->capacity() - buf->size(); callback(buf->buffer()); // this is the only one, so we can reuse it if(buffers.size() == 1) { + // Trick to make SACK work. If a lot of data was cleared up + // it means the local sequence number is much farther behind + // the real one + seq = buf->end_seq() - rem; + buf->reset(seq); + //printf("size=1, reset rem=%u start=%u end=%u\n", + // rem, buf->start_seq(), buf->end_seq()); break; } // to make it simple, just get rid of it and have the @@ -67,11 +77,27 @@ namespace tcp { // maybe it isnt even necessary..) else { - buffers.pop_front(); - buf = buffers.front().get(); + // if there are other buffers following this one, + // and the buffer wasnt full, fill the small gap + if(UNLIKELY(rem != 0)) + { + buf->reset(seq, rem); + //printf("remaining=%u, reset start=%u end=%u\n", + // rem, buf->start_seq(), buf->end_seq()); + Ensures(buf->end_seq() == buffers.at(1)->start_seq()); + } + else + { + //printf("finished, pop front start=%u end=%u\n", + // buf->start_seq(), buf->end_seq()); + buffers.pop_front(); + buf = buffers.front().get(); + } } - } - } + + } // < while(buf->is_ready() and buf == buffers.front().get()) + + } // < while(n) Ensures(not buffers.empty()); return recv; @@ -86,35 +112,42 @@ namespace tcp { return ptr.get(); } + //printf("seq=%u do not fit in any current buffer\n", seq); // We can still create a new one if(buffers.size() < buffer_limit) { // current cap - const auto& back = buffers.back(); + const auto& cur_back = buffers.back(); + //printf("current back, start=%u end=%u sz=%u\n", + // cur_back->start_seq(), cur_back->end_seq(), cur_back->size()); // TODO: if the gap is bigger than 1 buffer // we probably need to create multiple buffers, // ... or just decide we only support gaps of 1 buffer size. buffers.push_back( - std::make_unique(back->capacity(), back->end_seq())); + std::make_unique(cur_back->capacity(), cur_back->end_seq())); - //printf("new buffer added,fits(%lu)=%lu\n", - // seq, buffers.back()->fits(seq)); + auto& back = buffers.back(); + //printf("new buffer added start=%u end=%u, fits(%lu)=%lu\n", + // back->start_seq(), back->end_seq(), seq, back->fits(seq)); - if(buffers.back()->fits(seq) > 0) - return buffers.back().get(); + if(back->fits(seq) > 0) + return back.get(); } + //printf("did not fit\n"); return nullptr; } size_t Read_request::fits(const seq_t seq) const { + auto len = 0; // There is room in a existing one for(auto& ptr : buffers) { - if(ptr->fits(seq) > 0) - return ptr->fits(seq); + len += ptr->fits(seq); + /*if(ptr->fits(seq) > 0) + return ptr->fits(seq);*/ } // Possible to create one (or more) to support this? @@ -125,10 +158,10 @@ namespace tcp { const auto cap = back->capacity(); if(rel < cap) - return (cap - rel); + len += (cap - rel); } - return 0; + return len; } size_t Read_request::size() const @@ -160,9 +193,16 @@ namespace tcp { // get the first buffer auto* buf = it->get(); - // if it contains data without any holes, + + // okay, so here's the deal. + // there might be data in the read_buffer (due to liveupdate restore) + // so we need to be able to set a callback, then have reset called, + // to flush the data to the user. + + // if noone is using the buffer right now, (stupid yes) + // AND it contains data without any holes, // return it to the user - if(buf->size() > 0 and buf->missing() == 0) + if(buf->buffer().unique() and buf->size() > 0 and buf->missing() == 0) { callback(buf->buffer()); } From 1f915d4aa3c799930cbb337e051066a5d0bd62bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 9 Feb 2018 10:10:21 +0100 Subject: [PATCH 049/723] tcp: More readable in state SEQ check --- src/net/tcp/connection_states.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/net/tcp/connection_states.cpp b/src/net/tcp/connection_states.cpp index e4445638d3..a73f4a77c2 100644 --- a/src/net/tcp/connection_states.cpp +++ b/src/net/tcp/connection_states.cpp @@ -96,7 +96,7 @@ using namespace std; bool Connection::State::check_seq(Connection& tcp, const Packet& in) { auto& tcb = tcp.tcb(); - + uint32_t packet_end = static_cast(in.seq() + in.tcp_data_length()-1); // RFC 7323 Option::opt_ts* ts = nullptr; static constexpr uint8_t HEADER_WITH_TS{sizeof(Header) + 12}; @@ -131,12 +131,12 @@ bool Connection::State::check_seq(Connection& tcp, const Packet& in) goto acceptable; } // #3 (INVALID) - Packet is outside the right edge of the recv window - else if( in.seq() + in.tcp_data_length()-1 > tcb.RCV.NXT+tcb.RCV.WND ) { + else if( packet_end > tcb.RCV.NXT+tcb.RCV.WND ) { goto unacceptable; } // #4 - Packet with payload is what we expect or bigger, but inside our window - else if( tcb.RCV.NXT <= in.seq()+in.tcp_data_length()-1 - and in.seq()+in.tcp_data_length()-1 < tcb.RCV.NXT+tcb.RCV.WND ) { + else if( tcb.RCV.NXT <= packet_end + and packet_end < tcb.RCV.NXT+tcb.RCV.WND ) { goto acceptable; } /* From d69ce55fc3e212fcda8451dbbd213009e50d6e44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 9 Feb 2018 10:12:29 +0100 Subject: [PATCH 050/723] tcp: SACK support for recv data, send SACK opt --- api/net/tcp/connection.hpp | 2 + api/net/tcp/sack.hpp | 48 ++++++++----- src/net/tcp/connection.cpp | 138 +++++++++++++++++++++++++++++-------- 3 files changed, 141 insertions(+), 47 deletions(-) diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index 2f8a2c140a..f43a5bf7b3 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -1046,6 +1046,8 @@ class Connection { */ void recv_data(const Packet& in); + void recv_out_of_order(const Packet& in); + /** * @brief Acknowledge incoming data. This is done by: * - Trying to send data if possible (can send) diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index ecfb2267c4..f698350c17 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -92,8 +92,8 @@ inline std::ostream& operator<<(std::ostream& out, const Block& b) { using Entries = Fixed_vector; struct Ack_result { - Entries entries; - uint32_t bytes; + size_t length; + uint32_t blocksize; }; template @@ -105,15 +105,18 @@ class List { : impl{std::forward(args)} {} - Ack_result recv_out_of_order(const seq_t seq, const size_t len) - { return impl.recv_out_of_order({seq, static_cast(seq+len)}); } + Ack_result recv_out_of_order(const seq_t seq, size_t len) + { return impl.recv_out_of_order(seq, len); } - Ack_result new_valid_ack(const seq_t end) - { return impl.new_valid_ack(end); } + Ack_result new_valid_ack(const seq_t end, size_t len) + { return impl.new_valid_ack(end, len); } size_t size() const noexcept { return impl.size(); } + Entries recent_entries() const noexcept + { return impl.recent_entries(); } + List_impl impl; }; @@ -155,8 +158,12 @@ class Fixed_list { static_assert(N <= 32 && N > 0, "N wrong sized - optimized for small N"); - Ack_result recv_out_of_order(Block blk) + Ack_result recv_out_of_order(const seq_t seq, size_t len) { + // TODO: This just assumes nothing of the block exists from before. + // Uncertain if this will cause an issue. + Block blk{seq, static_cast(seq+len)}; + auto connected = connects_to(blocks.begin(), blocks.end(), blk); if (connected.end != blocks.end()) // Connectes to an end @@ -182,41 +189,38 @@ class Fixed_list { Expects(blocks.size() <= capacity); if(UNLIKELY(blocks.size() == capacity)) - return {recent_entries(), 0}; + return {0, 0}; blocks.push_front(blk); } - return {recent_entries(), blk.size()}; + // just return the full length + return {len, blk.size()}; } - Ack_result new_valid_ack(const seq_t seq) + Ack_result new_valid_ack(const seq_t seq, size_t len) { + const auto ack = seq + len; uint32_t bytes_freed = 0; for(auto it = blocks.begin(); it != blocks.end(); it++) { - if (it->start == seq) + if (it->contains(ack)) { bytes_freed = it->size(); + len -= (ack - it->start); blocks.erase(it); break; } } - return {recent_entries(), bytes_freed}; + return {len, bytes_freed}; } size_t size() const noexcept { return blocks.size(); } - void move_to_front(List_iterator it) - { - if(it != blocks.begin()) - blocks.splice(blocks.begin(), blocks, it); - } - - Entries recent_entries() const + Entries recent_entries() const noexcept { Entries ret; for(auto it = blocks.begin(); it != blocks.end() and ret.size() < ret.capacity(); it++) @@ -225,6 +229,12 @@ class Fixed_list { return ret; } + void move_to_front(List_iterator it) + { + if(it != blocks.begin()) + blocks.splice(blocks.begin(), blocks, it); + } + List blocks; }; diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 2e4aab328f..07b711458e 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -24,6 +24,14 @@ #include #include #include +#include // remove me, sack debugging +std::ostream& operator<< (std::ostream& out, const net::tcp::sack::Entries& ent) { + for (auto el : ent) { + out << el << "\n"; + } + return out; +} + using namespace net::tcp; using namespace std; @@ -71,6 +79,7 @@ void Connection::_on_read(size_t recv_bufsz, ReadCallback cb) // read request is already set, only reset if new size. else { + //printf("on_read already set\n"); read_request->callback = cb; // this will flush the current data to the user (if any) read_request->reset(recv_bufsz, seq_t(this->cb.RCV.NXT)); @@ -328,7 +337,21 @@ Packet_ptr Connection::create_outgoing_packet() if(cb.SND.TS_OK) packet->add_tcp_option_aligned(host_.get_ts_value(), cb.get_ts_recent()); - // Set SEQ and ACK - I think this is OK.. + + // Add SACK option (if any entries) + if(UNLIKELY(sack_list and sack_list->size())) + { + Expects(sack_perm); + + auto entries = sack_list->recent_entries(); + // swap to network endian before adding to packet + for(auto& ent : entries) + ent.swap_endian(); + + packet->add_tcp_option_aligned(entries); + } + + // Set SEQ and ACK packet->set_seq(cb.SND.NXT).set_ack(cb.RCV.NXT); debug(" Outgoing packet created: %s \n", packet->to_string().c_str()); @@ -669,53 +692,114 @@ void Connection::recv_data(const Packet& in) { Expects(in.has_tcp_data()); - const auto length = in.tcp_data_length(); + // Keep track if a packet is being sent during the async read callback + const auto snd_nxt = cb.SND.NXT; // The packet we expect if(cb.RCV.NXT == in.seq()) { + size_t length = in.tcp_data_length(); + + //auto& front = read_request->front(); + //if(front.missing()) { + // printf(" In order: SEQ=%u hole=%u fits=%u sz=%u front_start=%u front_end=%u\n", + // in.seq(), front.missing(), read_request->fits(in.seq()), read_request->size(), front.start_seq(), front.end_seq()); + //} + // If we had packet loss before (and SACK is on) // we need to clear up among the blocks // and increase the total amount of bytes acked if(UNLIKELY(sack_list)) { - auto res = sack_list->new_valid_ack(in.seq()); - cb.RCV.NXT += res.bytes; + //printf("In order: SEQ=%u sz=%lu - front: hole=%u fits=%u sz=%u cap=%u start=%u end=%u\n", + // in.seq(), length, front.missing(), read_request->fits(in.seq()), + // front.size(), front.capacity(), front.start_seq(), front.end_seq()); + + auto res = sack_list->new_valid_ack(in.seq(), length); + // if any bytes are cleared up in sack, increase expected sequence number + cb.RCV.NXT += res.blocksize; + + if(UNLIKELY(length != res.length)) + printf("mismatch len=%u res.len=%u\n", length, res.length); + // TODO: This Ensures verifies that we never have partial packets + Ensures(length == res.length); + //length = res.length; + + if(cb.RCV.NXT != in.seq()) + printf("ACKED res.bytes=%u\n", res.blocksize); + //std::cout << sack_list->recent_entries(); } - cb.RCV.NXT += length; + const auto recv = read_request->insert(in.seq(), in.tcp_data(), length, in.isset(PSH)); + Ensures(recv == length); + + cb.RCV.NXT += recv; } // Packet out of order - else + else if((in.seq() - cb.RCV.NXT) < cb.RCV.WND) { - // We know that out of order packets wouldnt reach here - // without SACK being permitted - Expects(sack_perm); + recv_out_of_order(in); + } - printf(" Out-of-order: RCV.NXT=%u SEQ=%u\n", - cb.RCV.NXT, in.seq()); + // User callback didnt result in transmitting an ACK + if(cb.SND.NXT == snd_nxt) + ack_data(); - // The SACK list is initated on the first out of order packet - if(not sack_list) - sack_list = std::make_unique(); + // [RFC 5681] ??? +} - auto res = sack_list->recv_out_of_order(in.seq(), length); - } +void Connection::recv_out_of_order(const Packet& in) +{ + // Packets before this point would totally ruin the buffer + Expects((in.seq() - cb.RCV.NXT) < cb.RCV.WND); + // We know that out of order packets wouldnt reach here + // without SACK being permitted + Expects(sack_perm); + + // The SACK list is initated on the first out of order packet + if(UNLIKELY(not sack_list)) + sack_list = std::make_unique(); + + const size_t length = in.tcp_data_length(); + auto seq = in.seq(); + auto fits = read_request->fits(seq); + + //if(fits) + // printf("Out-of-order: SEQ=%u REL=%u RCV.NXT=%u fits=%u\n", + // in.seq(), in.seq() - cb.RCV.NXT, cb.RCV.NXT, fits); + + // TODO: if our packet partial fits, we just ignores it for now + // to avoid headache + if(fits >= length) + { + auto inserted = read_request->insert(seq, in.tcp_data(), length, in.isset(PSH)); + Ensures(inserted == length); - // Keep track if a packet is being sent during the async read callback - const auto snd_nxt = cb.SND.NXT; + // Assume for now that we have room in sack list + auto res = sack_list->recv_out_of_order(seq, inserted); - if(read_request) - { - auto recv = receive(in.seq(), in.tcp_data(), length, in.isset(PSH)); - Ensures(recv == length); + Ensures(res.length == length); + + std::cout << sack_list->recent_entries(); } - // User callback didnt result in transmitting an ACK - if(cb.SND.NXT == snd_nxt) - ack_data(); + /* + size_t rem = length; + // Support filling partial packets + while(rem != 0 and fits != 0) + { + auto offset = length - rem; + auto inserted = read_request->insert(seq, in.tcp_data()+offset, std::min(rem, fits), in.isset(PSH)); - // [RFC 5681] ??? + // Assume for now that we have room in sack list + auto res = sack_list->recv_out_of_order(seq, inserted); + std::cout << res.entries; + + seq += inserted; + rem -= inserted; + + fits = read_request->fits(seq); + }*/ } void Connection::ack_data() @@ -1109,8 +1193,6 @@ void Connection::parse_options(const Packet& packet) { case Option::SACK_PERM: { - printf("\n"); - if(UNLIKELY(option->length != sizeof(Option::opt_sack_perm))) throw TCPBadOptionException{Option::SACK_PERM, "length != 2"}; From 50de688fca55cb892998104c037be15736cf9dbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 9 Feb 2018 10:19:14 +0100 Subject: [PATCH 051/723] test: Update TCP SACK unit test --- src/net/tcp/connection.cpp | 7 +++-- test/net/unit/tcp_sack_test.cpp | 52 ++++++++++++++++----------------- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 07b711458e..5e96750085 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -24,13 +24,14 @@ #include #include #include -#include // remove me, sack debugging + +/*#include // remove me, sack debugging std::ostream& operator<< (std::ostream& out, const net::tcp::sack::Entries& ent) { for (auto el : ent) { out << el << "\n"; } return out; -} +}*/ using namespace net::tcp; @@ -780,7 +781,7 @@ void Connection::recv_out_of_order(const Packet& in) Ensures(res.length == length); - std::cout << sack_list->recent_entries(); + //std::cout << sack_list->recent_entries(); } /* diff --git a/test/net/unit/tcp_sack_test.cpp b/test/net/unit/tcp_sack_test.cpp index 857790334c..f59e531f14 100644 --- a/test/net/unit/tcp_sack_test.cpp +++ b/test/net/unit/tcp_sack_test.cpp @@ -129,31 +129,31 @@ CASE("SACK Fixed List implementation [RFC 2018]") // 5000 5500 6000 res = sack_list.recv_out_of_order(5500, 500); - EXPECT(res.entries == expected({{5500,6000}})); + EXPECT(sack_list.recent_entries() == expected({{5500,6000}})); // 5000 5500 6500 res = sack_list.recv_out_of_order(6000, 500); - EXPECT(res.entries == expected({{5500,6500}})); + EXPECT(sack_list.recent_entries() == expected({{5500,6500}})); // 5000 5500 7000 res = sack_list.recv_out_of_order(6500, 500); - EXPECT(res.entries == expected({{5500,7000}})); + EXPECT(sack_list.recent_entries() == expected({{5500,7000}})); // 5000 5500 7500 res = sack_list.recv_out_of_order(7000,500); - EXPECT(res.entries == expected({{5500,7500}})); + EXPECT(sack_list.recent_entries() == expected({{5500,7500}})); // 5000 5500 8000 res = sack_list.recv_out_of_order(7500,500); - EXPECT(res.entries == expected({{5500,8000}})); + EXPECT(sack_list.recent_entries() == expected({{5500,8000}})); // 5000 5500 8500 res = sack_list.recv_out_of_order(8000,500); - EXPECT(res.entries == expected({{5500,8500}})); + EXPECT(sack_list.recent_entries() == expected({{5500,8500}})); // 5000 5500 9000 res = sack_list.recv_out_of_order(8500,500); - EXPECT(res.entries == expected({{5500, 9000}})); + EXPECT(sack_list.recent_entries() == expected({{5500, 9000}})); /* Case 3: The 2nd, 4th, 6th, and 8th (last) segments are @@ -180,15 +180,15 @@ CASE("SACK Fixed List implementation [RFC 2018]") // 5500 6000 6500 res = sack_list.recv_out_of_order(6000, 500); - EXPECT(res.entries == expected({{6000,6500}})); + EXPECT(sack_list.recent_entries() == expected({{6000,6500}})); // 5500 7000 7500 6000 6500 res = sack_list.recv_out_of_order(7000, 500); - EXPECT(res.entries == expected({{7000,7500}, {6000,6500}})); + EXPECT(sack_list.recent_entries() == expected({{7000,7500}, {6000,6500}})); // 5500 8000 8500 7000 7500 6000 6500 res = sack_list.recv_out_of_order(8000, 500); - EXPECT(res.entries == expected({{8000,8500}, {7000,7500}, {6000,6500}})); + EXPECT(sack_list.recent_entries() == expected({{8000,8500}, {7000,7500}, {6000,6500}})); /* Suppose at this point, the 4th packet is received out of order. @@ -208,7 +208,7 @@ CASE("SACK Fixed List implementation [RFC 2018]") // 6000 7500 8000 8500 res = sack_list.recv_out_of_order(6500, 500); - EXPECT(res.entries == expected({{6000,7500}, {8000,8500}})); + EXPECT(sack_list.recent_entries() == expected({{6000,7500}, {8000,8500}})); /* Suppose at this point, the 2nd segment is received. The data @@ -220,27 +220,27 @@ CASE("SACK Fixed List implementation [RFC 2018]") 5500 7500 8000 8500 */ - res = sack_list.new_valid_ack(5500 + 500); + res = sack_list.new_valid_ack(5500, 500); - EXPECT(res.entries == expected({{8000,8500}})); - EXPECT(res.bytes == 7500-6000); + EXPECT(sack_list.recent_entries() == expected({{8000,8500}})); + EXPECT(res.blocksize == 7500-6000); // Create a hole which connects both ends and fill the hole sack_list = Sack_list(); res = sack_list.recv_out_of_order(5500, 500); - EXPECT(res.entries == expected({{5500,6000}})); + EXPECT(sack_list.recent_entries() == expected({{5500,6000}})); res = sack_list.recv_out_of_order(6500, 500); - EXPECT(res.entries == expected({{6500,7000}, {5500,6000}})); + EXPECT(sack_list.recent_entries() == expected({{6500,7000}, {5500,6000}})); res = sack_list.recv_out_of_order(6000, 500); - EXPECT(res.entries == expected({{5500,7000}})); + EXPECT(sack_list.recent_entries() == expected({{5500,7000}})); - res = sack_list.new_valid_ack(5500); - EXPECT(res.entries == expected({})); - EXPECT(res.bytes == 1500); + res = sack_list.new_valid_ack(5000, 500); + EXPECT(sack_list.recent_entries() == expected({})); + EXPECT(res.blocksize == 1500); } CASE("SACK block list is full") @@ -265,29 +265,29 @@ CASE("SACK block list is full") } // Fully populated sack list - EXPECT(res.entries == expected({{seq - incr, (seq - incr ) + blksz}, + EXPECT(sack_list.recent_entries() == expected({{seq - incr, (seq - incr ) + blksz}, {seq - incr * 2, (seq - incr * 2) + blksz }, {seq - incr * 3, (seq - incr * 3) + blksz } })); - EXPECT(res.bytes == blksz); + EXPECT(res.blocksize == blksz); // Try adding one more res = sack_list.recv_out_of_order(seq, blksz); // We should now get the same - EXPECT(res.entries == expected({{seq - incr, (seq - incr ) + blksz}, + EXPECT(sack_list.recent_entries() == expected({{seq - incr, (seq - incr ) + blksz}, {seq - incr * 2, (seq - incr * 2) + blksz }, {seq - incr * 3, (seq - incr * 3) + blksz } })); // Nothing inserted - EXPECT(res.bytes == 0); + EXPECT(res.blocksize == 0); // Add a block that connects to the end, which should free up one spot res = sack_list.recv_out_of_order(seq - incr + blksz, blksz); - EXPECT(res.bytes == blksz); + EXPECT(res.blocksize == blksz); // Last block should now be larger - EXPECT(res.entries == expected({{seq - incr, (seq - incr ) + blksz + blksz }, + EXPECT(sack_list.recent_entries() == expected({{seq - incr, (seq - incr ) + blksz + blksz }, {seq - incr * 2, (seq - incr * 2) + blksz }, {seq - incr * 3, (seq - incr * 3) + blksz } })); From 5734ab8a1b4ba5a3d561aeb0d18bea8ec268df41 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 9 Feb 2018 11:32:50 +0100 Subject: [PATCH 052/723] alloc_lstack: fix include, remove spam --- src/musl/mmap.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 35c320f098..e4f35f7c00 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include extern uintptr_t heap_begin; extern uintptr_t heap_end = 0; @@ -13,9 +13,6 @@ using Alloc = util::alloc::Lstack<4096>; static Alloc alloc; void init_mmap(uintptr_t addr_begin){ - if (alloc.empty()){ - printf("Alloc not empty: %s\n ", alloc.begin() == nullptr ? "nullptr" : "size not 0"); - } Expects(alloc.empty()); auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); alloc.donate((void*)aligned_begin, (OS::heap_max() - aligned_begin) & ~(Alloc::align - 1)); From a6185394389e1da49658b0de3ce456e7daa2144c Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 9 Feb 2018 14:36:29 +0100 Subject: [PATCH 053/723] linker: remove crtbegin.o/crtend.o and .ctors/.dtors --- cmake/cross_compiled_libraries.cmake | 5 ---- cmake/post.service.cmake | 4 +-- etc/create_binary_bundle.sh | 13 -------- src/arch/x86_64/linker.ld | 44 ---------------------------- src/crt/c_abi.c | 20 ------------- 5 files changed, 1 insertion(+), 85 deletions(-) diff --git a/cmake/cross_compiled_libraries.cmake b/cmake/cross_compiled_libraries.cmake index 65988020c6..6f8325cb89 100644 --- a/cmake/cross_compiled_libraries.cmake +++ b/cmake/cross_compiled_libraries.cmake @@ -88,9 +88,6 @@ add_library(libm STATIC IMPORTED) set_target_properties(libm PROPERTIES IMPORTED_LOCATION ${MUSL_LIB_DIR}/libm.a) add_dependencies(libm PrecompiledLibraries) -set(CRTEND ${PRECOMPILED_DIR}/crt/crtend.o) -set(CRTBEGIN ${PRECOMPILED_DIR}/crt/crtbegin.o) - # # Installation # @@ -100,8 +97,6 @@ install(DIRECTORY ${LIBUNWIND_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include install(DIRECTORY ${MUSL_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/musl) -install(FILES ${CRTEND} ${CRTBEGIN} DESTINATION includeos/${ARCH}/lib) - set(CHAINLOAD_LOC ${CMAKE_CURRENT_BINARY_DIR}/precompiled/src/PrecompiledLibraries/chainloader) install(FILES ${CHAINLOAD_LOC} DESTINATION includeos) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index af7badc69d..1807dc2517 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -261,12 +261,11 @@ if ("${PLATFORM}" STREQUAL "x86_solo5") set(PRE_BSS_SIZE "--defsym PRE_BSS_AREA=0x200000") endif() -set(LDFLAGS "-nostdlib -melf_${ELF} --eh-frame-hdr ${STRIP_LV} --script=${INSTALL_LOC}/${ARCH}/linker.ld ${PRE_BSS_SIZE} ${INSTALL_LOC}/${ARCH}/lib/crtbegin.o") +set(LDFLAGS "-nostdlib -melf_${ELF} --eh-frame-hdr ${STRIP_LV} --script=${INSTALL_LOC}/${ARCH}/linker.ld ${PRE_BSS_SIZE}") set_target_properties(service PROPERTIES LINK_FLAGS "${LDFLAGS}") set(CRTN "${INSTALL_LOC}/${ARCH}/lib/crtn.o") -set(CRTEND "${INSTALL_LOC}/${ARCH}/lib/crtend.o") set(CRTI "${INSTALL_LOC}/${ARCH}/lib/crti.o") target_link_libraries(service ${CRTI}) @@ -461,7 +460,6 @@ target_link_libraries(service musl_syscalls - ${CRTEND} ${CRTN} ) # write binary location to known file diff --git a/etc/create_binary_bundle.sh b/etc/create_binary_bundle.sh index 72b75ecf30..859c92f6f1 100755 --- a/etc/create_binary_bundle.sh +++ b/etc/create_binary_bundle.sh @@ -162,19 +162,6 @@ function do_build { cp $include_libunwind/libunwind.h $BUNDLE_LOC/libunwind/include cp $include_libunwind/__libunwind_config.h $BUNDLE_LOC/libunwind/include - if [ $ARCH == "i686" ] - then - crtbegin=`$CC -m32 --print-file-name crtbegin.o` - crtend=`$CC -m32 --print-file-name crtend.o` - else - crtbegin=`$CC --print-file-name crtbegin.o` - crtend=`$CC --print-file-name crtend.o` - fi - - mkdir -p $BUNDLE_LOC/crt - cp $crtbegin $BUNDLE_LOC/crt/ - cp $crtend $BUNDLE_LOC/crt/ - } for B_ARCH in $BUNDLE_ARCHES diff --git a/src/arch/x86_64/linker.ld b/src/arch/x86_64/linker.ld index eba07c0a5e..559585e718 100644 --- a/src/arch/x86_64/linker.ld +++ b/src/arch/x86_64/linker.ld @@ -65,50 +65,6 @@ SECTIONS *(.got*) } -/** - * .ctors, .dtors, .preinit_array, .init_array, .fini_array - * from GNU LD default linker script - */ - -.ctors : - { - _GCONSTR_START_ = .; - /* gcc uses crtbegin.o to find the start of - the constructors, so we make sure it is - first. Because this is a wildcard, it - doesn't matter if the user does not - actually link against crtbegin.o; the - linker won't look for a file to match a - wildcard. The wildcard also means that it - doesn't matter which directory crtbegin.o - is in. */ - KEEP (*crtbegin.o(.ctors)) - KEEP (*crtbegin?.o(.ctors)) - /* We don't want to include the .ctor section from - the crtend.o file until after the sorted ctors. - The .ctor section from the crtend file contains the - end of ctors marker and it must be last */ - KEEP (*(EXCLUDE_FILE (*crtend.o *crtend?.o ) .ctors)) - KEEP (*(SORT(.ctors.*))) - KEEP (*(.ctors)) - _GCONSTR_END_ = .; - } - - .dtors : - { - KEEP (*crtbegin.o(.dtors)) - KEEP (*crtbegin?.o(.dtors)) - KEEP (*(EXCLUDE_FILE (*crtend.o *crtend?.o ) .dtors)) - KEEP (*(SORT(.dtors.*))) - KEEP (*(.dtors)) - } - - .preinit_array : - { - PROVIDE_HIDDEN (__preinit_array_start = .); - KEEP (*(.preinit_array)) - PROVIDE_HIDDEN (__preinit_array_end = .); - } .init_array : { diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index f3c1df8673..7643181577 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -34,26 +34,6 @@ void* __dso_handle; const uintptr_t __stack_chk_guard = (uintptr_t) _STACK_GUARD_VALUE_; extern void panic(const char* why) __attribute__((noreturn)); -extern void (*__preinit_array_start [])(); -extern void (*__preinit_array_end [])(); -extern void (*__init_array_start [])(); -extern void (*__init_array_end [])(); - -void __libc_init_array() { - size_t count, i; - - count = __preinit_array_end - __preinit_array_start; - for (i = 0; i < count; i++) - __preinit_array_start[i](); - - kprintf("Initializing C constructors \n"); - _init(); - kprintf("Initializing C++ constructors \n"); - count = __init_array_end - __init_array_start; - for (i = 0; i < count; i++) - __init_array_start[i](); -} - uint32_t _move_symbols(void* sym_loc) { extern char _ELF_SYM_START_; From b2038981ab40cc80051ff9ffb96c5ad5f7c1f146 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Feb 2018 10:19:54 +0100 Subject: [PATCH 054/723] posix: add back fd, socket, stat --- src/posix/fd.cpp | 60 ++++++++ src/posix/sys/socket.cpp | 186 ++++++++++++++++++++++++ src/posix/sys/stat.cpp | 307 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 553 insertions(+) create mode 100644 src/posix/fd.cpp create mode 100644 src/posix/sys/socket.cpp create mode 100644 src/posix/sys/stat.cpp diff --git a/src/posix/fd.cpp b/src/posix/fd.cpp new file mode 100644 index 0000000000..138afda1b5 --- /dev/null +++ b/src/posix/fd.cpp @@ -0,0 +1,60 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +int FD::fcntl(int cmd, va_list list) +{ + PRINT("fcntl(%d)\n", cmd); + switch (cmd) { + case F_GETFD: + // return descriptor flags + return dflags; + case F_SETFD: + // set desc flags from va_list + dflags = va_arg(list, int); + return 0; + case F_GETFL: + // return file access flags + return fflags; + case F_SETFL: + // set file access flags + fflags = va_arg(list, int); + return 0; + case F_DUPFD: + case F_DUPFD_CLOEXEC: + default: + errno = EINVAL; + return -1; + } +} +int FD::getsockopt(int fd, int, void *__restrict__, socklen_t *__restrict__) +{ + PRINT("getsockopt(%d) = -1\n", fd); + errno = ENOTSOCK; + return -1; +} +int FD::setsockopt(int fd, int, const void *, socklen_t) +{ + PRINT("setsockopt(%d) = -1\n", fd); + errno = ENOTSOCK; + return -1; +} diff --git a/src/posix/sys/socket.cpp b/src/posix/sys/socket.cpp new file mode 100644 index 0000000000..1cc28aa1e5 --- /dev/null +++ b/src/posix/sys/socket.cpp @@ -0,0 +1,186 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +int socket(int domain, int type, int protocol) +{ + PRINT("socket(%d, %d, %d)\n", domain, type, protocol); + // disallow strange domains, like ALG + if (domain < 0 || domain > AF_INET6) { errno = EAFNOSUPPORT; return -1; } + // disallow RAW etc + if (type < 0 || type > SOCK_DGRAM) { errno = EINVAL; return -1; } + // we are purposefully ignoring the protocol argument + if (protocol < 0) { errno = EPROTONOSUPPORT; return -1; } + + return [](const int type)->int{ + switch(type) + { + case SOCK_STREAM: + return FD_map::_open().get_id(); + case SOCK_DGRAM: + return FD_map::_open().get_id(); + default: + errno = EINVAL; + return -1; + } + }(type); +} + +int connect(int socket, const struct sockaddr *address, socklen_t len) +{ + PRINT("connect(%d, %p, %d)\n", socket, address, len); + try { + auto& fd = FD_map::_get(socket); + return fd.connect(address, len); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} + +ssize_t send(int socket, const void *message, size_t len, int fmt) +{ + PRINT("send(%d, %p, %lu, %d)\n", socket, message, len, fmt); + try { + auto& fd = FD_map::_get(socket); + return fd.send(message, len, fmt); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} +ssize_t sendto(int socket, const void *message, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t dest_len) +{ + PRINT("sendto(%d, %p, %lu, %d, %p, %d)\n", socket, message, len, flags, dest_addr, dest_len); + try { + auto& fd = FD_map::_get(socket); + return fd.sendto(message, len, flags, dest_addr, dest_len); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} +ssize_t recv(int socket, void *buffer, size_t length, int flags) +{ + PRINT("recv(%d, %p, %lu, %d)\n", socket, buffer, length, flags); + try { + auto& fd = FD_map::_get(socket); + return fd.recv(buffer, length, flags); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} +ssize_t recvfrom(int socket, void *buffer, size_t length, + int flags, struct sockaddr *address, socklen_t *address_len) +{ + try { + auto& fd = FD_map::_get(socket); + return fd.recvfrom(buffer, length, flags, address, address_len); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} +int listen(int socket, int backlog) +{ + PRINT("listen(%d, %d)\n", socket, backlog); + try { + auto& fd = FD_map::_get(socket); + return fd.listen(backlog); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} +int accept(int socket, struct sockaddr *address, socklen_t *len) +{ + PRINT("accept(%d, %p, %d)\n", socket, address, *len); + try { + auto& fd = FD_map::_get(socket); + return fd.accept(address, len); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} +int bind(int socket, const struct sockaddr* address, socklen_t len) +{ + PRINT("bind(%d, %p, %d)\n", socket, address, len); + try { + auto& fd = FD_map::_get(socket); + return fd.bind(address, len); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} +int getsockopt(int socket, int level, int option_name, + void *option_value, socklen_t *option_len) +{ + PRINT("getsockopt(%d, %d, %d, %p, %d)\n", socket, level, + option_name, option_value, option_len); + try { + auto& fd = FD_map::_get(socket); + return fd.getsockopt(level, option_name, option_value, option_len); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} +int setsockopt(int socket, int level, int option_name, + const void *option_value, socklen_t option_len) +{ + PRINT("setsockopt(%d, %d, %d, %p, %d)\n", socket, level, + option_name, option_value, option_len); + try { + auto& fd = FD_map::_get(socket); + return fd.setsockopt(level, option_name, option_value, option_len); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} +int shutdown(int socket, int how) +{ + PRINT("shutdown(%d, %d)\n", socket, how); + try { + auto& fd = FD_map::_get(socket); + return fd.shutdown(how); + } catch (const FD_not_found&) { + errno = EBADF; + return -1; + } +} + +bool validate_sockaddr_in(const struct sockaddr* address, socklen_t len) +{ + // The specified address is not a valid address for the address family of the specified socket. + if (UNLIKELY(len < sizeof(struct sockaddr_in))) { + PRINT("sockaddr_in length too short: %u vs %lu\n", len, sizeof(struct sockaddr_in)); + return false; + } + + return true; +} diff --git a/src/posix/sys/stat.cpp b/src/posix/sys/stat.cpp new file mode 100644 index 0000000000..5891150254 --- /dev/null +++ b/src/posix/sys/stat.cpp @@ -0,0 +1,307 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef DEFAULT_UMASK +#define DEFAULT_UMASK 002 +#endif + +extern fs::Disk_ptr& fs_disk(); +inline unsigned round_up(unsigned n, unsigned div) { + Expects(div > 0); + return (n + div - 1) / div; +} + +int chmod(const char *path, mode_t mode) { + (void) path; + (void) mode; + errno = EROFS; + return -1; +} + +int fchmod(int fildes, mode_t mode) +{ + try { + auto& fd = FD_map::_get(fildes); + return fd.fchmod(mode); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; + } +} + +int fchmodat(int filedes, const char *path, mode_t mode, int flag) +{ + try { + auto& fd = FD_map::_get(filedes); + return fd.fchmodat(path, mode, flag); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; + } +} + +int fstatat(int filedes, const char *path, struct stat *buf, int flag) +{ + if (filedes == AT_FDCWD) + { + char cwd_buf[PATH_MAX]; + char abs_path[PATH_MAX]; + if (getcwd(cwd_buf, PATH_MAX)) { + snprintf(abs_path, PATH_MAX, "%s/%s", cwd_buf, path); + } + return stat(abs_path, buf); + } + else + { + try { + auto& fd = FD_map::_get(filedes); + return fd.fstatat(path, buf, flag); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; + } + } +} + +int futimens(int filedes, const struct timespec times[2]) +{ + try { + auto& fd = FD_map::_get(filedes); + return fd.futimens(times); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; + } +} + +int utimensat(int filedes, const char *path, const struct timespec times[2], int flag) +{ + try { + auto& fd = FD_map::_get(filedes); + return fd.utimensat(path, times, flag); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; + } +} + +int mkdir(const char *path, mode_t mode) +{ + (void) path; + (void) mode; + errno = EROFS; + return -1; +} + +int mkdirat(int filedes, const char *path, mode_t mode) +{ + try { + auto& fd = FD_map::_get(filedes); + return fd.mkdirat(path, mode); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; + } +} + +int mkfifo(const char *path, mode_t mode) +{ + (void) path; + (void) mode; + errno = EROFS; + return -1; +} + +int mkfifoat(int filedes, const char *path, mode_t mode) +{ + try { + auto& fd = FD_map::_get(filedes); + return fd.mkfifoat(path, mode); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; + } +} + +int mknod(const char *path, mode_t mode, dev_t dev) +{ + (void) path; + (void) mode; + (void) dev; + errno = EROFS; + return -1; +} + +int mknodat(int filedes, const char *path, mode_t mode, dev_t dev) +{ + try { + auto& fd = FD_map::_get(filedes); + return fd.mknodat(path, mode, dev); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; + } +} + +int stat(const char *path, struct stat *buf) +{ + if (buf == nullptr) + { + PRINT("stat(%s, %p) == %d\n", path, buf, -1); + errno = EFAULT; + return -1; + } + memset(buf, 0, sizeof(struct stat)); + try { + auto ent = fs::VFS::stat_sync(path); + if (ent.is_valid()) + { + if (ent.is_file()) buf->st_mode = S_IFREG; + if (ent.is_dir()) buf->st_mode = S_IFDIR; + buf->st_dev = ent.device_id(); + buf->st_ino = ent.block(); + buf->st_nlink = 1; + buf->st_size = ent.size(); + buf->st_atime = ent.modified(); + buf->st_ctime = ent.modified(); + buf->st_mtime = ent.modified(); + buf->st_blocks = buf->st_size > 0 ? round_up(buf->st_size, 512) : 0; + buf->st_blksize = fs::MemDisk::SECTOR_SIZE; + PRINT("stat(%s, %p) == %d\n", path, buf, 0); + return 0; + } + else { + PRINT("stat(%s, %p) == %d\n", path, buf, -1); + errno = EIO; + return -1; + } + } + catch (...) + { + PRINT("stat(%s, %p) == %d\n", path, buf, -1); + errno = EIO; + return -1; + } +} +int lstat(const char *path, struct stat *buf) +{ + // NOTE: should stat symlinks, instead of following them + return stat(path, buf); +} + +mode_t umask(mode_t cmask) +{ + (void) cmask; + return DEFAULT_UMASK; +} + +int fstat(int fd, struct stat* stat_buf) +{ + if (fd < 0) { + PRINT("fstat(%d, %p) == %d\n", fd, stat_buf, -1); + errno = EBADF; + return -1; + } + if (stat_buf == nullptr) { + PRINT("fstat(%d, %p) == %d\n", fd, stat_buf, -1); + errno = EINVAL; + return -1; + } + if (fd < 4 || fd == 4) + { + if (fd == 4) { + PRINT("fstat(%d, %p) == %d\n", fd, stat_buf, 0); + } + stat_buf->st_dev = 6; + stat_buf->st_ino = fd; + stat_buf->st_mode = 0x21b6; + stat_buf->st_nlink = 1; + stat_buf->st_uid = 0; + stat_buf->st_gid = 0; + stat_buf->st_rdev = 265; + stat_buf->st_size = 0; + if (fd == 4) + { + stat_buf->st_atime = RTC::now(); + stat_buf->st_mtime = RTC::now(); + stat_buf->st_ctime = RTC::now(); + } + stat_buf->st_blksize = 4096; + stat_buf->st_blocks = 0; + return 0; + } + PRINT("fstat(%d, %p) == %d\n", fd, stat_buf, -1); + errno = ENOSYS; + return -1; +} + +extern "C" +int __xstat(int ver, const char * path, struct stat * stat_buf) +{ + PRINT("__xstat(%d, '%s', %p)\n", ver, path, stat_buf); + errno = ENOSYS; + return -1; +} + +extern "C" +int __lxstat(int ver, const char * path, struct stat * stat_buf) +{ + PRINT("__lxstat(%d, '%s', %p)\n", ver, path, stat_buf); + errno = ENOSYS; + return -1; +} + +#define LINUX_STAT_VER 1 + +extern "C" +int __fxstat(int ver, int fd, struct stat * stat_buf) +{ + PRINT("__fxstat(%d, %d, %p)\n", ver, fd, stat_buf); + if (fd < 0) { + errno = EBADF; + return -1; + } + if (stat_buf == nullptr) { + errno = EINVAL; + return -1; + } + if (ver != LINUX_STAT_VER) { + errno = EINVAL; + return -1; + } + + return fstat(fd, stat_buf); +} From 2500a54b0e1e7bf7a744be04d650ed8ecfd5eb06 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Feb 2018 18:20:58 +0100 Subject: [PATCH 055/723] cmake: use C++17 --- CMakeLists.txt | 6 +++--- src/drivers/heap_debugging.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 51d86dfa9f..7e683b5871 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,7 +29,7 @@ set(BIN ${CMAKE_INSTALL_PREFIX}/includeos/bin) set(SCRIPTS ${CMAKE_INSTALL_PREFIX}/includeos/scripts) # C++ version -set(CMAKE_CXX_STANDARD 14) +set(CPP_VERSION c++17) # create OS version string from git describe (used in CXX flags) execute_process(COMMAND git describe --dirty @@ -139,11 +139,11 @@ endif() # initialize C and C++ compiler flags if (CMAKE_COMPILER_IS_GNUCC) # gcc/g++ settings - set(CMAKE_CXX_FLAGS " -MMD ${CAPABS} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c ${LIBCPP_THREADING} -DOS_VERSION=\\\"${OS_VERSION}\\\"") + set(CMAKE_CXX_FLAGS " -MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c ${LIBCPP_THREADING} -DOS_VERSION=\\\"${OS_VERSION}\\\"") set(CMAKE_C_FLAGS " -MMD ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c -DOS_VERSION=\"\"${OS_VERSION}\"\"") else() # these kinda work with llvm - set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c ${LIBCPP_THREADING} -DOS_VERSION=\\\"${OS_VERSION}\\\"") + set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c ${LIBCPP_THREADING} -DOS_VERSION=\\\"${OS_VERSION}\\\"") set(CMAKE_C_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c -DOS_VERSION=\"\"${OS_VERSION}\"\"") endif() diff --git a/src/drivers/heap_debugging.cpp b/src/drivers/heap_debugging.cpp index b1470ab02a..3f14ce12ae 100644 --- a/src/drivers/heap_debugging.cpp +++ b/src/drivers/heap_debugging.cpp @@ -99,7 +99,7 @@ static allocation* find_alloc(char* addr) return nullptr; } -void* operator new (std::size_t len) throw(std::bad_alloc) +void* operator new (std::size_t len) { void* data = nullptr; @@ -159,7 +159,7 @@ void* operator new (std::size_t len) throw(std::bad_alloc) return data; } } -void* operator new[] (std::size_t n) throw(std::bad_alloc) +void* operator new[] (std::size_t n) { return ::operator new (n); } From 6cdadf4cab78ae16bcbf8ec6d17acf61f6cf3fd0 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Feb 2018 18:22:24 +0100 Subject: [PATCH 056/723] musl: remove unused mmap2.cpp --- src/musl/CMakeLists.txt | 2 +- src/musl/mmap2.cpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 src/musl/mmap2.cpp diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index 5fc6514805..ee8b4cd97e 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -5,7 +5,7 @@ set(MUSL_OBJECTS lseek.cpp sched_getaffinity.cpp sysinfo.cpp prlimit64.cpp getrlimit.cpp sched_yield.cpp set_robust_list.cpp nanosleep.cpp open.cpp clock_gettime.cpp gettimeofday.cpp - poll.cpp mmap2.cpp exit_group.cpp exit.cpp close.cpp set_tid_address.cpp + poll.cpp exit_group.cpp exit.cpp close.cpp set_tid_address.cpp pipe.cpp read.cpp readv.cpp getpid.cpp mknod.cpp stat.cpp sync.cpp msync.cpp mincore.cpp syscall_n.cpp sigmask.cpp gettid.cpp tkill.cpp ) diff --git a/src/musl/mmap2.cpp b/src/musl/mmap2.cpp deleted file mode 100644 index 7a1adb1edf..0000000000 --- a/src/musl/mmap2.cpp +++ /dev/null @@ -1 +0,0 @@ -#include "common.hpp" From 3b034f3acf8da7cb1493ab15ad43e25fbeec8e97 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Feb 2018 18:34:54 +0100 Subject: [PATCH 057/723] musl: strace mechanism --- src/include/fd.hpp | 3 ++ src/musl/brk.cpp | 17 ++++---- src/musl/close.cpp | 12 +++--- src/musl/common.hpp | 59 ++++++++++++++++++++++++++-- src/musl/fcntl.cpp | 7 +++- src/musl/futex.cpp | 20 +++++----- src/musl/getpid.cpp | 7 +++- src/musl/getrlimit.cpp | 14 +++++-- src/musl/gettid.cpp | 15 ++++--- src/musl/ioctl.cpp | 41 +++++++++++-------- src/musl/lseek.cpp | 32 +++++++++++---- src/musl/mmap.cpp | 32 +++++++-------- src/musl/open.cpp | 16 ++++++-- src/musl/readv.cpp | 27 ++++--------- src/musl/set_tid_address.cpp | 13 +++--- src/musl/sigmask.cpp | 53 +++++++++++++++---------- src/musl/stub.hpp | 16 ++++++++ src/musl/syscall_n.cpp | 9 +++-- src/musl/tkill.cpp | 9 +++-- src/musl/write.cpp | 32 +++++++++++++-- src/musl/writev.cpp | 12 +++--- src/platform/x86_pc/kernel_start.cpp | 2 + src/posix/fd.cpp | 8 ++++ 23 files changed, 309 insertions(+), 147 deletions(-) create mode 100644 src/musl/stub.hpp diff --git a/src/include/fd.hpp b/src/include/fd.hpp index 77ff39ea80..03991ee9ea 100644 --- a/src/include/fd.hpp +++ b/src/include/fd.hpp @@ -20,6 +20,8 @@ #define INCLUDE_FD_HPP #include +#include +#include #include /** @@ -40,6 +42,7 @@ class FD { virtual int write(const void*, size_t) { return -1; } virtual int close() = 0; virtual int fcntl(int, va_list); + virtual int ioctl(int, void*); /** SOCKET **/ virtual int accept(struct sockaddr *__restrict__, socklen_t *__restrict__) { return -1; } diff --git a/src/musl/brk.cpp b/src/musl/brk.cpp index 7f04b29575..10928e656b 100644 --- a/src/musl/brk.cpp +++ b/src/musl/brk.cpp @@ -31,29 +31,28 @@ void _init_heap(uintptr_t free_mem_begin) } - -extern "C" -uintptr_t syscall_SYS_brk(void* addr) +static uintptr_t sys_brk(void* addr) { - STRACE("syscall brk. Heap begin=0x%lx, OS::heap_max=0x%lx, addr=%p" - "\n\t requests %i bytes \n", - heap_begin, OS::heap_max(), addr, (uintptr_t)addr - brk_end); if (addr == nullptr or (uintptr_t)addr > heap_begin + __brk_max or (uintptr_t)addr < heap_begin) { - kprintf("\tBrk failed\n"); return brk_end; } brk_end = (uintptr_t)addr; if (brk_end > brk_init) { - kprintf("\t Initializing %i b\n", brk_end - brk_init); memset((void*)brk_init, 0, brk_end - brk_init); brk_init = brk_end; } - kprintf("\t Done, returning 0x%lx\n", brk_end); return brk_end; } + + +extern "C" +uintptr_t syscall_SYS_brk(void* addr) +{ + return strace(sys_brk, "brk", addr); +} diff --git a/src/musl/close.cpp b/src/musl/close.cpp index 40466a61f7..58690a9e33 100644 --- a/src/musl/close.cpp +++ b/src/musl/close.cpp @@ -1,10 +1,12 @@ #include "common.hpp" -extern "C" -int syscall_SYS_close(int fd) { - STRACE("close fd=%i\n", fd); - +static int sys_close(int fd) { if (!fd) return -1; - return 0; + +}; + +extern "C" +int syscall_SYS_close(int fd) { + return strace(sys_close, "close", fd); } diff --git a/src/musl/common.hpp b/src/musl/common.hpp index 3c3a78b7f8..6431bd4959 100644 --- a/src/musl/common.hpp +++ b/src/musl/common.hpp @@ -1,7 +1,60 @@ #pragma once -#include +#include +#include +#include -#define STUB(X) kprintf(" stubbed syscall %s called\n", X) +#define STUB(X) printf(" stubbed syscall %s called\n", X) -#define STRACE(X, ...) kprintf(" " X, ##__VA_ARGS__) +#ifndef ENABLE_STRACE +#define ENABLE_STRACE false +#endif + +constexpr bool __strace = ENABLE_STRACE; +extern "C" void __serial_print(const char*, size_t); + +template +inline constexpr void pr_param(std::ostream& out){ + +} + +template +inline constexpr auto& pr_param(std::ostream& out, L lhs, Args&&... rest){ + if constexpr (sizeof...(rest) > 0) + { + out << lhs << ", "; + pr_param(out, rest...); + } else { + out << lhs; + } + return out; +} + +template +inline void strace_print(const char* name, Ret ret, Args&&... args){ + extern bool __libc_initialized; + + if (not __libc_initialized) + return; + + std::stringstream out; + out << name << "("; + pr_param(out, args...); + out << ") = " << ret; + if (errno) + out << " " << strerror(errno); + out << '\n'; + auto str = out.str(); + __serial_print(str.data(), str.size()); +} + +// strace, calling the syscall, recording return value and printing if enabled +template +inline auto strace(Fn func, const char* name, Args&&... args) { + auto ret = func(args...); + + if constexpr (__strace) + strace_print(name, ret, args...); + + return ret; +} diff --git a/src/musl/fcntl.cpp b/src/musl/fcntl.cpp index 764a7a0403..a3fce61335 100644 --- a/src/musl/fcntl.cpp +++ b/src/musl/fcntl.cpp @@ -1,8 +1,11 @@ #include "common.hpp" +static int sys_fcntl(int fd, int cmd, ...){ + return 0; +} + extern "C" int syscall_SYS_fcntl(int fd, int cmd, ... /* arg */ ) { - STRACE("fcntl(%d, %x) = 0\n", fd, cmd); - return 0; + return strace(sys_fcntl, "fcntl", fd, cmd); } diff --git a/src/musl/futex.cpp b/src/musl/futex.cpp index e0aa66685c..10534c6000 100644 --- a/src/musl/futex.cpp +++ b/src/musl/futex.cpp @@ -1,5 +1,6 @@ -#include "common.hpp" +#include "stub.hpp" #include +#include #define FUTEX_WAIT 0 #define FUTEX_WAKE 1 @@ -11,22 +12,15 @@ #define FUTEX_UNLOCK_PI 7 #define FUTEX_TRYLOCK_PI 8 #define FUTEX_WAIT_BITSET 9 - #define FUTEX_PRIVATE 128 - #define FUTEX_CLOCK_REALTIME 256 -extern "C" void panic(char*); extern void print_backtrace(); -extern "C" -int syscall_SYS_futex(int *uaddr, int futex_op, int val, +static int sys_futex(int *uaddr, int futex_op, int val, const struct timespec *timeout, int val3) { - STRACE("syscall futex: uaddr=%p futex_op=0x%x val=%i timeout=%p, val3: %i\n", - uaddr, futex_op, val, timeout, val3); - //print_backtrace(); if (*uaddr != val){ return EAGAIN; } else { @@ -37,6 +31,12 @@ int syscall_SYS_futex(int *uaddr, int futex_op, int val, kprintf("No timeout\n"); } - kprintf("Futex ok\n"); return 0; } + +extern "C" +int syscall_SYS_futex(int *uaddr, int futex_op, int val, + const struct timespec *timeout, int val3) +{ + return stubtrace(sys_futex, "futex", uaddr, futex_op, val, timeout, val3); +} diff --git a/src/musl/getpid.cpp b/src/musl/getpid.cpp index 25469c1571..c2d70e3932 100644 --- a/src/musl/getpid.cpp +++ b/src/musl/getpid.cpp @@ -1,7 +1,10 @@ #include "common.hpp" +int sys_getpid() { + return 1; +} + extern "C" long syscall_SYS_getpid() { - STRACE("getpid"); - return 0; + return strace(sys_getpid, "getpid"); } diff --git a/src/musl/getrlimit.cpp b/src/musl/getrlimit.cpp index 84a813ff5b..ab8b6edef9 100644 --- a/src/musl/getrlimit.cpp +++ b/src/musl/getrlimit.cpp @@ -1,7 +1,13 @@ -#include "common.hpp" +#include + +#include "stub.hpp" + +static int sys_getrlimit(int resource, struct rlimit *rlim) { + errno = ENOSYS; + return -1; +} extern "C" -long syscall_SYS_getrlimit() { - STUB("getrlimit"); - return 0; +long syscall_SYS_getrlimit(int resource, struct rlimit *rlim) { + return stubtrace(sys_getrlimit, "getrlimit", resource, rlim); } diff --git a/src/musl/gettid.cpp b/src/musl/gettid.cpp index 63129ba9c3..1e9cdebbc5 100644 --- a/src/musl/gettid.cpp +++ b/src/musl/gettid.cpp @@ -1,11 +1,14 @@ #include "common.hpp" +static long sys_gettid() { +#ifdef INCLUDEOS_SINGLE_THREADED + return 0; +#else +#error "gettid not implemented for threaded IncludeOS" +#endif +} + extern "C" long syscall_SYS_gettid() { - STRACE("gettid"); - #ifdef INCLUDEOS_SINGLE_THREADED - return 0; - #else - #error "gettid not implemented for threaded IncludeOS" - #endif + return strace(sys_gettid, "gettid"); } diff --git a/src/musl/ioctl.cpp b/src/musl/ioctl.cpp index 4f5b0de890..660b134e7c 100644 --- a/src/musl/ioctl.cpp +++ b/src/musl/ioctl.cpp @@ -1,26 +1,35 @@ -#include "common.hpp" #include +#include -#warning stub +#include "stub.hpp" -extern "C" -int syscall_SYS_ioctl(int fd, int req, void* arg) { - STRACE("syscall ioctl: fd=%i, req=0x%x\n", - fd, req); +static int sys_ioctl(int fd, int req, void* arg) { + if (fd == 1 or fd == 2) { + if (req == TIOCGWINSZ) { + winsize* ws = (winsize*)arg; + ws->ws_row = 25; + ws->ws_col = 80; + return 0; + } + + if (req == TIOCSWINSZ) { + winsize* ws = (winsize*)arg; + } - if (req == TIOCGWINSZ) { - kprintf("\t* Wants to get winsize\n"); - winsize* ws = (winsize*)arg; - ws->ws_row = 25; - ws->ws_col = 80; return 0; } - if (req == TIOCSWINSZ) { - winsize* ws = (winsize*)arg; - kprintf("\t* Wants to set winsize to %i x %i\n", - ws->ws_row, ws->ws_col); + try { + auto& fildes = FD_map::_get(fd); + return fildes.ioctl(req, arg); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; } +} - return 0; +extern "C" +int syscall_SYS_ioctl(int fd, int req, void* arg) { + return stubtrace(sys_ioctl, "ioctl", fd, req, arg); } diff --git a/src/musl/lseek.cpp b/src/musl/lseek.cpp index 16907da960..1d975e32f7 100644 --- a/src/musl/lseek.cpp +++ b/src/musl/lseek.cpp @@ -1,13 +1,31 @@ -#include "common.hpp" +#include +#include +#include + +#include "stub.hpp" + +off_t sys_lseek(int fd, off_t offset, int whence) { + errno = ENOSYS; + return -1; +} + +off_t sys__llseek(unsigned int fd, unsigned long offset_high, + unsigned long offset_low, loff_t *result, + unsigned int whence) { + errno = ENOSYS; + return -1; +} + extern "C" -long syscall_SYS_lseek() { - STUB("lseek"); - return 0; +off_t syscall_SYS_lseek(int fd, off_t offset, int whence) { + return stubtrace(sys_lseek, "lseek", fd, offset, whence); } extern "C" -long syscall_SYS__llseek() { - STUB("_lseek"); - return 0; +int syscall_SYS__llseek(unsigned int fd, unsigned long offset_high, + unsigned long offset_low, loff_t *result, + unsigned int whence) { + return stubtrace(sys__llseek, "_llseek", fd, offset_high, offset_low, + result, whence); } diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index e4f35f7c00..914d8ec6be 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -19,24 +19,21 @@ void init_mmap(uintptr_t addr_begin){ } -extern "C" + +extern "C" __attribute__((weak)) void* __kalloc(size_t size){ return alloc.allocate(size); } -extern "C" +extern "C" __attribute__((weak)) void __kfree (void* ptr, size_t size){ alloc.deallocate(ptr, size); } -extern "C" -void* syscall_SYS_mmap(void *addr, size_t length, int prot, int flags, +static void* sys_mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) { - STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d \n", - addr, length, prot, flags, fd, offset); - // TODO: Mapping to file descriptor if (fd > 0) { assert(false && "Mapping to file descriptor not yet implemented"); @@ -48,13 +45,15 @@ void* syscall_SYS_mmap(void *addr, size_t length, int prot, int flags, return MAP_FAILED; } - void* res = __kalloc(length); - STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d res=%p\n", - addr, length, prot, flags, fd, offset, res); - - return res; + return __kalloc(length);; } +extern "C" +void* syscall_SYS_mmap(void *addr, size_t length, int prot, int flags, + int fd, off_t offset) +{ + return strace(sys_mmap, "mmap", addr, length, prot, flags, fd, offset); +} /** The mmap2() system call provides the same interface as mmap(2), @@ -65,12 +64,9 @@ void* syscall_SYS_mmap(void *addr, size_t length, int prot, int flags, http://man7.org/linux/man-pages/man2/mmap2.2.html **/ + extern "C" void* syscall_SYS_mmap2(void *addr, size_t length, int prot, - int flags, int fd, off_t offset) { - uintptr_t res = heap_begin + current_pos; - current_pos += length; - STRACE("syscall mmap2: addr=%p len=%u prot=%d fl=%d fd=%d off=%d\n", - addr, length, prot, flags, fd, offset); - return (void*)res; + int flags, int fd, off_t offset) { + return strace(sys_mmap, "mmap2", addr, length, prot, flags, fd, offset * 4096); } diff --git a/src/musl/open.cpp b/src/musl/open.cpp index fa1c8a16e4..03817690b1 100644 --- a/src/musl/open.cpp +++ b/src/musl/open.cpp @@ -1,13 +1,21 @@ -#include "common.hpp" +#include "stub.hpp" #include +int sys_open(const char *pathname, int flags, mode_t mode = 0) { + return -1; +} + +int sys_creat(const char *pathname, mode_t mode) { + return -1; +} + extern "C" { int syscall_SYS_open(const char *pathname, int flags, mode_t mode = 0) { - STRACE("open pathname=%s, flags=0x%x, mode=0x%x", - pathname, flags, mode); + return stubtrace(sys_open, "open", pathname, flags, mode); } int syscall_SYS_creat(const char *pathname, mode_t mode) { - STUB("creat"); + return stubtrace(sys_creat, "creat", pathname, mode); } + } // extern "C" diff --git a/src/musl/readv.cpp b/src/musl/readv.cpp index afbdbc2cb2..ccab1d6638 100644 --- a/src/musl/readv.cpp +++ b/src/musl/readv.cpp @@ -1,26 +1,15 @@ -#include "common.hpp" +#include "stub.hpp" #include #include -extern "C" -ssize_t syscall_SYS_readv(int fd, const struct iovec *iov, int iovcnt) - +static ssize_t sys_readv(int fd, const struct iovec *iov, int iovcnt) { - /* - if(fd <= 2) - { - ssize_t res = 0; - for(int i = 0; i < iovcnt; i++) - { - auto* text = (const char*)iov[i].iov_base; - auto len = iov[i].iov_len; - kprintf("%.*s", len, text); - res += len; - } - return res; - }*/ - STRACE("syscall readv: fd=%d iov=%p iovcount=%d [base=%p sz=%d]\n", - fd, iov, iovcnt, iov->iov_base, iov->iov_len); errno = ENOSYS; return -1; } + +extern "C" +ssize_t syscall_SYS_readv(int fd, const struct iovec *iov, int iovcnt) +{ + return stubtrace(sys_readv, "readv", fd, iov, iovcnt); +} diff --git a/src/musl/set_tid_address.cpp b/src/musl/set_tid_address.cpp index b9b51f1469..64619decf5 100644 --- a/src/musl/set_tid_address.cpp +++ b/src/musl/set_tid_address.cpp @@ -12,13 +12,12 @@ struct { #warning syscall set_tid_address is not yet thread safe #endif -extern "C" -long syscall_SYS_set_tid_address(int* tidptr) { - STRACE("set_tid_address, %p\n", tidptr); - if (tidptr) { - kprintf("\t*tidptr: 0x%x\n", *tidptr); - } +static long sys_set_tid_address(int* tidptr) { __main_thread__.clear_child_tid = tidptr; - return __main_thread__.tid; } + +extern "C" +long syscall_SYS_set_tid_address(int* tidptr) { + return strace(sys_set_tid_address, "set_tid_address", tidptr); +} diff --git a/src/musl/sigmask.cpp b/src/musl/sigmask.cpp index fb32a635ee..1aa4c210dd 100644 --- a/src/musl/sigmask.cpp +++ b/src/musl/sigmask.cpp @@ -1,35 +1,48 @@ -#include "common.hpp" +#include "stub.hpp" #include -extern "C" { -long syscall_SYS_sigmask() { - STUB("sigmask"); +static int sys_rt_sigprocmask (int how, const sigset_t *set, sigset_t *oldset){ + return 0; +} + +static int sys_sigmask(int signum) +{ return 0; } -long syscall_SYS_rt_sigprocmask(int how, const sigset_t *set, sigset_t *oldset) { +extern const bool __strace; -char* howstr = nullptr; +extern "C" { +int syscall_SYS_rt_sigprocmask(int how, const sigset_t *set, sigset_t *oldset) { + + if constexpr (__strace) { -switch(how) { + char* howstr = nullptr; - case (SIG_BLOCK): -howstr="BLOCK"; -break; + switch(how) { - case (SIG_UNBLOCK): -howstr="UNBLOCK"; -break; + case (SIG_BLOCK): + howstr="BLOCK"; + break; - case (SIG_SETMASK): -howstr="SETMASK"; -break; + case (SIG_UNBLOCK): + howstr="UNBLOCK"; + break; + case (SIG_SETMASK): + howstr="SETMASK"; + break; + } + auto ret = sys_rt_sigprocmask(how, set, oldset); + stubtrace_print("sys_rt_sigprocmask", howstr, set, oldset); + return ret; + } + return sys_rt_sigprocmask(how, set, oldset); } -STRACE("rt_sigprocmask how=%i (%s), set=%p, oldset=%p\n", - how, howstr, set, oldset); -return 0; +long syscall_SYS_sigmask(int signum) { + return stubtrace(sys_sigmask, "sigmask", signum); +} + } -} // extern "C" diff --git a/src/musl/stub.hpp b/src/musl/stub.hpp new file mode 100644 index 0000000000..1b61f78b2d --- /dev/null +++ b/src/musl/stub.hpp @@ -0,0 +1,16 @@ +#include "common.hpp" + +template +inline void stubtrace_print(const char* name, R ret, Args&&... args) { + printf(" Stubbed syscall: "); + strace_print(name, ret, args...); +} + +// Strace for stubbed syscalls. +// calling the syscall, recording return value and always printing +template +inline auto stubtrace(Fn func, const char* name, Args&&... args) { + auto ret = func(args...); + stubtrace_print(name, ret, args...); + return ret; +} diff --git a/src/musl/syscall_n.cpp b/src/musl/syscall_n.cpp index 9ff6cd2ec3..8f569b1430 100644 --- a/src/musl/syscall_n.cpp +++ b/src/musl/syscall_n.cpp @@ -1,7 +1,10 @@ -#include "common.hpp" +#include "stub.hpp" + +long syscall(long i){ + return 0; +}; extern "C" long syscall_n(long i) { - STRACE("syscall_n, n=%i\n", i); - return 0; + return stubtrace(syscall, "syscall", i); } diff --git a/src/musl/tkill.cpp b/src/musl/tkill.cpp index 39c8990419..ba385431b0 100644 --- a/src/musl/tkill.cpp +++ b/src/musl/tkill.cpp @@ -1,12 +1,15 @@ #include "common.hpp" #include -extern "C" -int syscall_SYS_tkill(int tid, int sig) { - STRACE("tkill(%i, %i)", tid, sig); +static int sys_tkill(int tid, int sig) { #ifdef INCLUDEOS_SINGLE_THREADED exit(sig); #else #error "tkill not implemented for threaded IncludeOS" #endif } + +extern "C" +int syscall_SYS_tkill(int tid, int sig) { + return strace(sys_tkill, "tkill", tid, sig); +} diff --git a/src/musl/write.cpp b/src/musl/write.cpp index 8fae36001e..d707d9411a 100644 --- a/src/musl/write.cpp +++ b/src/musl/write.cpp @@ -1,10 +1,34 @@ +#include +#include +#include #include "common.hpp" +#include + extern "C" void __serial_print(const char*, size_t); + + +// The actual syscall +static long sys_write(int fd, char* str, size_t len) { + kprintf("Sys write\n"); + + if (fd <= 0) { + errno = EBADF; + return -1; + } + + if (fd == 1 or fd == 2) { + OS::print(str, len); + return len; + } + + // TODO: integrate with old file descriptors + errno = ENOSYS; + return -1; +} + +// The syscall wrapper, using strace if enabled extern "C" long syscall_SYS_write(int fd, char* str, size_t len) { - STUB("write"); - - __serial_print(str, len); - return 0; + return strace(sys_write, "write", fd, str, len); } diff --git a/src/musl/writev.cpp b/src/musl/writev.cpp index f0d382e54b..e81310ec8e 100644 --- a/src/musl/writev.cpp +++ b/src/musl/writev.cpp @@ -2,8 +2,7 @@ #include -extern "C" -ssize_t syscall_SYS_writev(int fd, const struct iovec *iov, int iovcnt) +static ssize_t sys_writev(int fd, const struct iovec *iov, int iovcnt) { if(fd <= 2) { @@ -12,12 +11,15 @@ ssize_t syscall_SYS_writev(int fd, const struct iovec *iov, int iovcnt) { auto* text = (const char*)iov[i].iov_base; auto len = iov[i].iov_len; - kprintf("%.*s", len, text); + OS::print(text, len); res += len; } return res; } - STRACE("syscall writev: fd=%d iov=%p iovcount=%d [base=%p sz=%d]\n", - fd, iov, iovcnt, iov->iov_base, iov->iov_len); return 0; } + +extern "C" +ssize_t syscall_SYS_writev(int fd, const struct iovec *iov, int iovcnt){ + return strace(sys_writev, "writev", fd, iov, iovcnt); +} diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index f258dcaaf4..e513c1774a 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -52,9 +52,11 @@ void _init_bss() uintptr_t __grub_magic = 0xc001; uintptr_t __grub_addr = 0x7001; +bool __libc_initialized = false; int kernel_main(int, char * *, char * *) { kprintf(" libc initialization complete \n"); + __libc_initialized = true; Expects(__tl1__ == 42); Elf_binary elf{{(char*)&_ELF_START_, &_ELF_END_ - &_ELF_START_}}; Expects(elf.is_ELF() && "ELF header intact"); diff --git a/src/posix/fd.cpp b/src/posix/fd.cpp index 138afda1b5..1351f2a18d 100644 --- a/src/posix/fd.cpp +++ b/src/posix/fd.cpp @@ -46,6 +46,14 @@ int FD::fcntl(int cmd, va_list list) return -1; } } + +int FD::ioctl(int req, void* arg) +{ + PRINT("ioctl(%d, %p) = -1\n", req, arg); + errno = ENOSYS; + return -1; +} + int FD::getsockopt(int fd, int, void *__restrict__, socklen_t *__restrict__) { PRINT("getsockopt(%d) = -1\n", fd); From ff2e4304d0126522e5bce85d943f07878831eff0 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 15 Feb 2018 16:27:06 +0100 Subject: [PATCH 058/723] arch: reorganized arch headers --- api/arch.hpp | 3 + api/arch/i686.hpp | 2 +- api/arch/x86.hpp | 30 ---- {src/platform/x86_pc => api/arch/x86}/cpu.hpp | 10 ++ {src/platform/x86_pc => api/arch/x86}/gdt.hpp | 10 +- api/arch/{x86_paging.hpp => x86/paging.hpp} | 0 .../paging_utils.hpp} | 0 api/arch/x86_64.hpp | 2 +- src/arch/x86_64/CMakeLists.txt | 3 +- ...{syscall_entry.asm => __syscall_entry.asm} | 0 src/arch/x86_64/paging.cpp | 4 +- src/arch/x86_64/syscall_entry.cpp | 64 +++++++++ src/musl/write.cpp | 1 - src/platform/kvm/kvmclock.cpp | 2 +- src/platform/kvm/pv_eoi.cpp | 2 +- src/platform/x86_pc/apic.cpp | 2 +- src/platform/x86_pc/gdt.cpp | 2 +- src/platform/x86_pc/kernel_start.cpp | 67 ++------- src/platform/x86_pc/platform.cpp | 130 ++++-------------- src/platform/x86_pc/x2apic.hpp | 2 +- src/platform/x86_pc/xapic.hpp | 2 +- 21 files changed, 127 insertions(+), 211 deletions(-) delete mode 100644 api/arch/x86.hpp rename {src/platform/x86_pc => api/arch/x86}/cpu.hpp (89%) rename {src/platform/x86_pc => api/arch/x86}/gdt.hpp (88%) rename api/arch/{x86_paging.hpp => x86/paging.hpp} (100%) rename api/arch/{x86_paging_utils.hpp => x86/paging_utils.hpp} (100%) rename src/arch/x86_64/{syscall_entry.asm => __syscall_entry.asm} (100%) create mode 100644 src/arch/x86_64/syscall_entry.cpp diff --git a/api/arch.hpp b/api/arch.hpp index 8b5a7d26c5..0bf8af3cb3 100644 --- a/api/arch.hpp +++ b/api/arch.hpp @@ -44,6 +44,9 @@ extern uint64_t __arch_system_time() noexcept; extern timespec __arch_wall_clock() noexcept; inline uint64_t __arch_cpu_cycles() noexcept; +inline void __arch_hw_barrier() noexcept { + __sync_synchronize(); +} inline void __sw_barrier() noexcept { diff --git a/api/arch/i686.hpp b/api/arch/i686.hpp index 3ca354edad..4a44ff5c0a 100644 --- a/api/arch/i686.hpp +++ b/api/arch/i686.hpp @@ -19,7 +19,7 @@ #ifndef i686_ARCH_HPP #define i686_ARCH_HPP -#include +#define ARCH_x86 inline uint64_t __arch_cpu_cycles() noexcept { uint64_t ret; diff --git a/api/arch/x86.hpp b/api/arch/x86.hpp deleted file mode 100644 index 6427e07370..0000000000 --- a/api/arch/x86.hpp +++ /dev/null @@ -1,30 +0,0 @@ -// -*-C++-*- -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef X86_ARCH_HPP -#define X86_ARCH_HPP - -#define ARCH_x86 - - -inline void __arch_hw_barrier() noexcept -{ - asm volatile("mfence" ::: "memory"); -} - -#endif diff --git a/src/platform/x86_pc/cpu.hpp b/api/arch/x86/cpu.hpp similarity index 89% rename from src/platform/x86_pc/cpu.hpp rename to api/arch/x86/cpu.hpp index 0e3cbfcf54..7e8376b1f2 100644 --- a/src/platform/x86_pc/cpu.hpp +++ b/api/arch/x86/cpu.hpp @@ -74,6 +74,16 @@ namespace x86 #error "write_msr() not implemented for selected arch" #endif } + +#if defined(ARCH_x86_64) + static inline void set_fs(void* entry) noexcept { + write_msr(IA32_FS_BASE, (uintptr_t) entry); + } + static inline void set_gs(void* entry) noexcept { + write_msr(IA32_GS_BASE, (uintptr_t) entry); + } +#endif + }; } diff --git a/src/platform/x86_pc/gdt.hpp b/api/arch/x86/gdt.hpp similarity index 88% rename from src/platform/x86_pc/gdt.hpp rename to api/arch/x86/gdt.hpp index 4a07606f4e..56d15d1662 100644 --- a/src/platform/x86_pc/gdt.hpp +++ b/api/arch/x86/gdt.hpp @@ -48,15 +48,7 @@ struct GDT static void reload_gdt(GDT& base) noexcept; -#if defined(ARCH_x86_64) - - static inline void set_fs(void* entry) noexcept { - CPU::write_msr(IA32_FS_BASE, (uintptr_t) entry); - } - static inline void set_gs(void* entry) noexcept { - CPU::write_msr(IA32_GS_BASE, (uintptr_t) entry); - } -#elif defined(ARCH_i686) +#if defined(ARCH_x86) static inline void set_fs(uint16_t entry) noexcept { asm volatile("movw %%ax, %%fs" : : "a"(entry * 0x8)); } diff --git a/api/arch/x86_paging.hpp b/api/arch/x86/paging.hpp similarity index 100% rename from api/arch/x86_paging.hpp rename to api/arch/x86/paging.hpp diff --git a/api/arch/x86_paging_utils.hpp b/api/arch/x86/paging_utils.hpp similarity index 100% rename from api/arch/x86_paging_utils.hpp rename to api/arch/x86/paging_utils.hpp diff --git a/api/arch/x86_64.hpp b/api/arch/x86_64.hpp index 41e1ebd4d5..6a7cb8d4df 100644 --- a/api/arch/x86_64.hpp +++ b/api/arch/x86_64.hpp @@ -19,7 +19,7 @@ #ifndef X86_64_ARCH_HPP #define X86_64_ARCH_HPP -#include +#define ARCH_x86 inline uint64_t __arch_cpu_cycles() noexcept { uint32_t hi, lo; diff --git a/src/arch/x86_64/CMakeLists.txt b/src/arch/x86_64/CMakeLists.txt index 7ef0860173..0bd7a9af0a 100644 --- a/src/arch/x86_64/CMakeLists.txt +++ b/src/arch/x86_64/CMakeLists.txt @@ -8,7 +8,8 @@ set(ARCH_OBJECTS exceptions.asm interrupts.asm fiber_asm.asm - syscall_entry.asm + __syscall_entry.asm + syscall_entry.cpp ist.cpp paging.cpp init_paging.cpp diff --git a/src/arch/x86_64/syscall_entry.asm b/src/arch/x86_64/__syscall_entry.asm similarity index 100% rename from src/arch/x86_64/syscall_entry.asm rename to src/arch/x86_64/__syscall_entry.asm diff --git a/src/arch/x86_64/paging.cpp b/src/arch/x86_64/paging.cpp index 0ffb1b32f2..77ce23c135 100644 --- a/src/arch/x86_64/paging.cpp +++ b/src/arch/x86_64/paging.cpp @@ -15,8 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. #include -#include -#include +#include +#include #include template<> diff --git a/src/arch/x86_64/syscall_entry.cpp b/src/arch/x86_64/syscall_entry.cpp new file mode 100644 index 0000000000..edf7b0b031 --- /dev/null +++ b/src/arch/x86_64/syscall_entry.cpp @@ -0,0 +1,64 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#define ARCH_SET_GS 0x1001 +#define ARCH_SET_FS 0x1002 +#define ARCH_GET_FS 0x1003 +#define ARCH_GET_GS 0x1004 + + +#ifdef __x86_64__ +int __arch_prctl(int code, uintptr_t ptr){ + + switch(code){ + case ARCH_SET_GS: + kprintf(" set_gs to %#lx\n", ptr); + if (!ptr) return -1; + x86::CPU::set_gs((void*)ptr); + break; + case ARCH_SET_FS: + kprintf(" set_fs to %#lx\n", ptr); + if (!ptr) return -1; + x86::CPU::set_fs((void*)ptr); + break; + case ARCH_GET_GS: + panic(" get gs \n"); + break; + case ARCH_GET_FS: + panic(" get gs \n"); + break; + } + return 0; +} +#endif + +extern "C" +uintptr_t syscall_entry(uint64_t n, uint64_t a1, uint64_t a2, uint64_t a3, + uint64_t a4, uint64_t a5) +{ + kprintf(" no %lu (a1=%#lx a2=%#lx a3=%#lx a4=%#lx a5=%#lx) \n", + n, a1, a2, a3, a4, a5); + + switch(n) { + case 158: // arch_prctl + __arch_prctl(a1, a2); + break; + } + return 0; +} diff --git a/src/musl/write.cpp b/src/musl/write.cpp index d707d9411a..1d72e1b4d6 100644 --- a/src/musl/write.cpp +++ b/src/musl/write.cpp @@ -10,7 +10,6 @@ extern "C" void __serial_print(const char*, size_t); // The actual syscall static long sys_write(int fd, char* str, size_t len) { - kprintf("Sys write\n"); if (fd <= 0) { errno = EBADF; diff --git a/src/platform/kvm/kvmclock.cpp b/src/platform/kvm/kvmclock.cpp index d5559773c9..f6dfe2d637 100644 --- a/src/platform/kvm/kvmclock.cpp +++ b/src/platform/kvm/kvmclock.cpp @@ -1,5 +1,5 @@ #include "kvmclock.hpp" -#include "../x86_pc/cpu.hpp" +#include #include #include #include diff --git a/src/platform/kvm/pv_eoi.cpp b/src/platform/kvm/pv_eoi.cpp index 1ac79d4f62..1f04877f2f 100644 --- a/src/platform/kvm/pv_eoi.cpp +++ b/src/platform/kvm/pv_eoi.cpp @@ -1,6 +1,6 @@ #include #include -#include "../x86_pc/cpu.hpp" +#include #include // *** manual *** diff --git a/src/platform/x86_pc/apic.cpp b/src/platform/x86_pc/apic.cpp index 5f8a8f16c6..b1752efc3f 100644 --- a/src/platform/x86_pc/apic.cpp +++ b/src/platform/x86_pc/apic.cpp @@ -18,7 +18,7 @@ #include "apic.hpp" #include "ioapic.hpp" #include "acpi.hpp" -#include "cpu.hpp" +#include #include "pic.hpp" #include "smp.hpp" #include diff --git a/src/platform/x86_pc/gdt.cpp b/src/platform/x86_pc/gdt.cpp index 7e0a4e26f1..f33b9328fc 100644 --- a/src/platform/x86_pc/gdt.cpp +++ b/src/platform/x86_pc/gdt.cpp @@ -1,4 +1,4 @@ -#include "gdt.hpp" +#include #include #include diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index e513c1774a..0a05ba42a1 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -21,15 +21,17 @@ #include #include #include -#include "cpu.hpp" -#include "gdt.hpp" +#include +#include +#include + +#include "idt.hpp" extern "C" { void __init_serial1(); void __init_sanity_checks(); void kernel_sanity_checks(); void _init_bss(); - void __test_syscall(void* p); uintptr_t _multiboot_free_begin(uintptr_t boot_addr); uintptr_t _move_symbols(uintptr_t loc); void _init_heap(uintptr_t free_mem_begin); @@ -40,7 +42,7 @@ extern char _ELF_END_; extern char _INIT_START_; extern char _FINI_START_; -thread_local int __tl1__ = 42; +static thread_local int __tl1__ = 42; void _init_bss() @@ -61,9 +63,11 @@ int kernel_main(int, char * *, char * *) { Elf_binary elf{{(char*)&_ELF_START_, &_ELF_END_ - &_ELF_START_}}; Expects(elf.is_ELF() && "ELF header intact"); + kprintf(" OS start \n"); // Initialize early OS, platform and devices OS::start(__grub_magic,__grub_addr); + kprintf(" post start \n"); // Initialize common subsystems and call Service::start OS::post_start(); @@ -84,56 +88,6 @@ void __init_tls(size_t* p) extern "C" int __libc_start_main(int (*main)(int,char **,char **), int argc, char **argv); -extern "C" void _init(); -extern "C" void _fini(); -#include -#include - - -#define ARCH_SET_GS 0x1001 -#define ARCH_SET_FS 0x1002 -#define ARCH_GET_FS 0x1003 -#define ARCH_GET_GS 0x1004 - -int __arch_prctl(int code, uintptr_t ptr){ - - switch(code){ - case ARCH_SET_GS: - kprintf(" set_gs to %#lx\n", ptr); - if (!ptr) return -1; - x86::GDT::set_gs((void*)ptr); - break; - case ARCH_SET_FS: - kprintf(" set_fs to %#lx\n", ptr); - if (!ptr) return -1; - x86::GDT::set_fs((void*)ptr); - break; - case ARCH_GET_GS: - panic(" get gs \n"); - break; - case ARCH_GET_FS: - panic(" get gs \n"); - break; - } - return 0; -} - - -extern "C" -uintptr_t syscall_entry(uint64_t n, uint64_t a1, uint64_t a2, uint64_t a3, - uint64_t a4, uint64_t a5) -{ - kprintf(" no %lu (a1=%#lx a2=%#lx a3=%#lx a4=%#lx a5=%#lx) \n", - n, a1, a2, a3, a4, a5); - - switch(n) { - case 158: // arch_prctl - __arch_prctl(a1, a2); - break; - } - return 0; -} - extern "C" uintptr_t __syscall_entry(); extern "C" @@ -176,12 +130,13 @@ void kernel_start(uintptr_t magic, uintptr_t addr) kprintf("* Init .bss\n"); _init_bss(); - kprintf("* Init heap\n"); _init_heap(free_mem_begin); - kprintf("* Thread local1: %i\n", __tl1__); + // Initialize CPU exceptions + x86::idt_initialize_for_cpu(0); + kprintf("* Thread local1: %i\n", __tl1__); kprintf("* Elf start: 0x%lx\n", &_ELF_START_); auto* ehdr = (Elf64_Ehdr*)&_ELF_START_; diff --git a/src/platform/x86_pc/platform.cpp b/src/platform/x86_pc/platform.cpp index 6eaba5c661..ca1ce4b06a 100644 --- a/src/platform/x86_pc/platform.cpp +++ b/src/platform/x86_pc/platform.cpp @@ -19,10 +19,10 @@ #include "apic.hpp" #include "apic_timer.hpp" #include "clocks.hpp" -#include "gdt.hpp" #include "idt.hpp" #include "smbios.hpp" #include "smp.hpp" +#include #include #include #include @@ -33,67 +33,49 @@ extern "C" char* get_cpu_esp(); extern "C" void* get_cpu_ebp(); -#define _SENTINEL_VALUE_ 0x123456789ABCDEF -namespace tls { - size_t get_tls_size(); - void fill_tls_data(char*); -} struct alignas(64) smp_table { - // thread self-pointer - void* tls_data; // 0x0 - // per-cpu cpuid (and more) + // per-cpu cpuid int cpuid; - int reserved; - -#ifdef ARCH_x86_64 - uintptr_t pad[3]; // 64-bit padding - uintptr_t guard; // _SENTINEL_VALUE_ -#else - uint32_t pad[2]; - uintptr_t guard; // _SENTINEL_VALUE_ - x86::GDT gdt; // 32-bit GDT -#endif /** put more here **/ }; -#ifdef ARCH_x86_64 -// FS:0x28 on Linux is storing a special sentinel stack-guard value -static_assert(offsetof(smp_table, guard) == 0x28, "Linux stack sentinel"); -#endif -using namespace x86; -namespace x86 { - void initialize_tls_for_smp(); -} +static SMP_ARRAY cpu_tables; void __platform_init() { // read ACPI tables - ACPI::init(); + x86::ACPI::init(); // read SMBIOS tables - SMBIOS::init(); - - // setup process tables - INFO("x86", "Setting up TLS"); - initialize_tls_for_smp(); + x86::SMBIOS::init(); // enable fs/gs for local APIC INFO("x86", "Setting up GDT, TLS, IST"); - initialize_gdt_for_cpu(0); + //initialize_gdt_for_cpu(0); #ifdef ARCH_x86_64 // setup Interrupt Stack Table x86::ist_initialize_for_cpu(0, 0x9D3F0); #endif // IDT manager: Interrupt and exception handlers - INFO("x86", "Creating CPU exception handlers"); - x86::idt_initialize_for_cpu(0); + //INFO("x86", "Creating CPU exception handlers"); + // x86::idt_initialize_for_cpu(0); + + cpu_tables[0].cpuid = 0; + +#ifdef ARCH_x86_64 + x86::CPU::set_gs(&cpu_tables[0]); +#else + x86::CPU::set_fs(&cpu_tables[0]); +#endif + + MYINFO("Creating CPU exception handlers"); Events::get(0).init_local(); // setup APIC, APIC timer, SMP etc. - APIC::init(); + x86::APIC::init(); // enable interrupts MYINFO("Enabling interrupts"); @@ -106,17 +88,17 @@ void __platform_init() // Setup kernel clocks MYINFO("Setting up kernel clock sources"); - Clocks::init(); + x86::Clocks::init(); if (OS::cpu_freq().count() <= 0.0) { - OS::cpu_khz_ = Clocks::get_khz(); + OS::cpu_khz_ = x86::Clocks::get_khz(); } INFO2("+--> %f MHz", OS::cpu_freq().count() / 1000.0); // Note: CPU freq must be known before we can start timer system // Initialize APIC timers and timer systems // Deferred call to Service::ready() when calibration is complete - APIC_Timer::calibrate(); + x86::APIC_Timer::calibrate(); // Initialize storage devices PCI_manager::init(PCI::STORAGE); @@ -131,80 +113,20 @@ void __platform_init() void __arch_enable_legacy_irq(uint8_t irq) { - APIC::enable_irq(irq); + x86::APIC::enable_irq(irq); } void __arch_disable_legacy_irq(uint8_t irq) { - APIC::disable_irq(irq); + x86::APIC::disable_irq(irq); } void __arch_poweroff() { - ACPI::shutdown(); + x86::ACPI::shutdown(); __builtin_unreachable(); } void __arch_reboot() { - ACPI::reboot(); + x86::ACPI::reboot(); __builtin_unreachable(); } - -#include -namespace x86 -{ - constexpr size_t smp_table_aligned_size() - { - size_t size = sizeof(smp_table); - if (size & 63) size += 64 - (size & 63); - return size; - } - - static std::vector tls_buffers; - - void initialize_tls_for_smp() - { - const size_t thread_size = tls::get_tls_size(); - const size_t total_size = thread_size + smp_table_aligned_size(); - const size_t cpu_count = ACPI::get_cpus().size(); - - //printf("TLS buffers are %lu bytes, SMP table %lu bytes\n", total_size, smp_table_aligned_size()); - char* buffer = (char*) memalign(64, total_size * cpu_count); - tls_buffers.reserve(cpu_count); - for (auto cpu = 0u; cpu < cpu_count; cpu++) - tls_buffers.push_back(&buffer[total_size * cpu]); - } - - void initialize_gdt_for_cpu(int cpu_id) - { - char* tls_data = tls_buffers.at(cpu_id); - char* tls_table = tls_data + tls::get_tls_size(); - // TLS data at front of buffer - tls::fill_tls_data(tls_data); - // SMP control block after TLS data - auto* table = (smp_table*) tls_table; - table->tls_data = tls_data; - table->cpuid = cpu_id; - table->guard = (uintptr_t) _SENTINEL_VALUE_; - // should be at least 8-byte aligned - assert((((uintptr_t) table) & 7) == 0); -#ifdef ARCH_x86_64 - //GDT::set_fs(table); // TLS self-ptr in fs - GDT::set_gs(&table->cpuid); // PER_CPU on gs -#else - // initialize GDT for this core - auto& gdt = table->gdt; - gdt.initialize(); - // create PER-CPU segment - int fs = gdt.create_data(&table->cpuid, 1); - // create per-thread segment - int gs = gdt.create_data(table, 0xffffffff); - // load GDT and refresh segments - GDT::reload_gdt(gdt); - // enable per-cpu and per-thread - //GDT::set_fs(fs); - GDT::set_gs(gs); -#endif - // hardware barrier - __sync_synchronize(); - } -} // x86 diff --git a/src/platform/x86_pc/x2apic.hpp b/src/platform/x86_pc/x2apic.hpp index 99353e5cfe..a510be77b7 100644 --- a/src/platform/x86_pc/x2apic.hpp +++ b/src/platform/x86_pc/x2apic.hpp @@ -21,7 +21,7 @@ #include "apic_iface.hpp" #include "apic_regs.hpp" -#include "cpu.hpp" +#include #include #include #include diff --git a/src/platform/x86_pc/xapic.hpp b/src/platform/x86_pc/xapic.hpp index 8716f8e197..b9ad03f6a0 100644 --- a/src/platform/x86_pc/xapic.hpp +++ b/src/platform/x86_pc/xapic.hpp @@ -21,7 +21,7 @@ #include "apic_iface.hpp" #include "apic_regs.hpp" -#include "cpu.hpp" +#include #include #include #include From 889370dbf81dd3c611efbbb3f3870c71b19c70db Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 16 Feb 2018 14:23:58 +0100 Subject: [PATCH 059/723] musl: graceful shutdown on exit + cleanup --- src/musl/exit.cpp | 16 +++++++++++----- src/musl/write.cpp | 7 ------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/musl/exit.cpp b/src/musl/exit.cpp index 6449602ab6..862cf36d60 100644 --- a/src/musl/exit.cpp +++ b/src/musl/exit.cpp @@ -1,9 +1,15 @@ #include "common.hpp" +#include +#include #include -extern "C" -long syscall_SYS_exit() { - STUB("exit"); - panic("Exiting\n"); - return 0; +static int sys_exit(int status){ + const std::string msg = "Service exited with status " + std::to_string(status) + "\n"; + OS::print(msg.data(), msg.size()); + __arch_poweroff(); +} + +extern "C" __attribute__((noreturn)) +void syscall_SYS_exit(int status) { + strace(sys_exit, "exit", status); } diff --git a/src/musl/write.cpp b/src/musl/write.cpp index 1d72e1b4d6..602f3171ef 100644 --- a/src/musl/write.cpp +++ b/src/musl/write.cpp @@ -1,12 +1,5 @@ -#include #include -#include #include "common.hpp" -#include - - -extern "C" void __serial_print(const char*, size_t); - // The actual syscall static long sys_write(int fd, char* str, size_t len) { From 20ab0f6c1d91d882a3d945cfc2e740fafb59b582 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 16 Feb 2018 16:24:06 +0100 Subject: [PATCH 060/723] SMP: refactor per cpu, remove old thread_local --- api/kernel/fiber.hpp | 9 +-- api/smp | 15 +++- src/kernel/events.cpp | 2 +- src/kernel/fiber.cpp | 39 +++++----- src/kernel/rng.cpp | 2 +- src/kernel/syscalls.cpp | 2 +- src/kernel/timers.cpp | 2 +- src/musl/gettid.cpp | 11 ++- src/musl/mmap.diff | 79 +++++++++++++++++++++ src/musl/set_tid_address.cpp | 10 +-- src/musl/tkill.cpp | 13 ++-- src/platform/kvm/kvmclock.cpp | 2 +- src/platform/x86_pc/apic_revenant.cpp | 14 ++-- src/platform/x86_pc/apic_revenant.hpp | 9 ++- src/platform/x86_pc/clocks.cpp | 2 +- src/platform/x86_pc/os.cpp | 2 +- src/platform/x86_pc/platform.cpp | 31 ++++---- src/platform/x86_pc/smp.cpp | 61 ++-------------- test/kernel/integration/fiber/fiber_smp.cpp | 11 +-- test/kernel/integration/fiber/service.cpp | 8 ++- test/kernel/integration/smp/service.cpp | 20 +++++- 21 files changed, 207 insertions(+), 137 deletions(-) create mode 100644 src/musl/mmap.diff diff --git a/api/kernel/fiber.hpp b/api/kernel/fiber.hpp index 864bd26471..16f3608d84 100644 --- a/api/kernel/fiber.hpp +++ b/api/kernel/fiber.hpp @@ -21,6 +21,7 @@ #include #include +#include #ifndef INCLUDEOS_SINGLE_THREADED #include @@ -199,10 +200,10 @@ class Fiber { } static Fiber* main() - { return main_; } + { return PER_CPU(main_); } static Fiber* current() - { return current_; } + { return PER_CPU(current_); } static int last_id() { @@ -222,8 +223,8 @@ class Fiber { #else static std::atomic next_id_; #endif - static thread_local Fiber* main_; - static thread_local Fiber* current_; + static SMP::Array main_; + static SMP::Array current_; // Uniquely identify return target (yield / exit) // first stack frame and yield will use this to identify next stack diff --git a/api/smp b/api/smp index 78e332f943..5722ed5426 100644 --- a/api/smp +++ b/api/smp @@ -21,6 +21,7 @@ #define API_SMP_HEADER #include +#include #include #ifndef SMP_MAX_CORES @@ -33,9 +34,6 @@ #define SMP_ALIGN 64 -template -using SMP_ARRAY = std::array; - // access a std::array of T by current CPU id #define PER_CPU(x) (per_cpu_help(x)) @@ -44,12 +42,23 @@ public: using task_func = delegate; using done_func = delegate; + template + using Array = std::array; + // return the current CPU id static int cpu_id() noexcept; // return the number of active CPUs static int cpu_count() noexcept; + // Return the indices of all initialized CPU cores + static const std::vector& active_cpus(); + + static int active_cpus(int index){ + return active_cpus().at(index); + } + + // implement this function to execute something on all APs at init static void init_task(); diff --git a/src/kernel/events.cpp b/src/kernel/events.cpp index 1adf500f73..31de5f0c71 100644 --- a/src/kernel/events.cpp +++ b/src/kernel/events.cpp @@ -22,7 +22,7 @@ #include //#define DEBUG_SMP -static SMP_ARRAY managers; +static SMP::Array managers; Events& Events::get(int cpuid) { diff --git a/src/kernel/fiber.cpp b/src/kernel/fiber.cpp index b4d6e6dfa6..918ec48539 100644 --- a/src/kernel/fiber.cpp +++ b/src/kernel/fiber.cpp @@ -29,9 +29,8 @@ int Fiber::next_id_{0}; std::atomic Fiber::next_id_{0}; #endif -thread_local void* caller_stack_ = nullptr; -thread_local Fiber* Fiber::main_ = nullptr; -thread_local Fiber* Fiber::current_ = nullptr; +SMP::Array Fiber::main_ {nullptr}; +SMP::Array Fiber::current_ {nullptr}; extern "C" { void __fiber_jumpstart(volatile void* th_stack, volatile Fiber* f, volatile void* parent_stack); @@ -45,7 +44,7 @@ extern "C" { f->ret_= f->func_(f->param_); // Last stackframe before switching back. Done. - Fiber::current_ = f->parent_ ? f->parent_ : nullptr; + PER_CPU(Fiber::current_) = f->parent_ ? f->parent_ : nullptr; f->done_ = true; } } @@ -58,17 +57,17 @@ void Fiber::start() { throw Err_bad_fiber("Can't start fiber without a function"); // Become main if none exists - if (not main_) - main_ = this; + if (not PER_CPU(main_)) + PER_CPU(main_) = this; // Suspend current running fiber if any - if (current_) { - current_ -> suspended_ = true; - make_parent(current_); + if (PER_CPU(current_)) { + PER_CPU(current_) -> suspended_ = true; + make_parent(PER_CPU(current_)); } // Become current - current_ = this; + PER_CPU(current_) = this; started_ = true; running_ = true; @@ -78,19 +77,17 @@ void Fiber::start() { // Returns here after first yield / final return - if (main_ == this) - main_ = nullptr; + if (PER_CPU(main_) == this) + PER_CPU(main_) = nullptr; - Expects(current_ != this); - - return; + Expects(PER_CPU(current_) != this); } void Fiber::yield() { - auto* from = current_; + auto* from = PER_CPU(current_); Expects(from); - auto* into = current_->parent_; + auto* into = PER_CPU(current_)->parent_; Expects(into); Expects(into->suspended()); Expects(into->stack_loc_); @@ -100,7 +97,7 @@ void Fiber::yield() { from->suspended_ = true; from->running_ = false; - current_ = into; + PER_CPU(current_) = into; __fiber_yield(from->parent_stack_, &(from->stack_loc_)); @@ -112,11 +109,11 @@ void Fiber::resume() if (not suspended_ or done_ or func_ == nullptr) return; - Expects(current_); + Expects(PER_CPU(current_)); - make_parent(current_); + make_parent(PER_CPU(current_)); - current_ = this; + PER_CPU(current_) = this; suspended_ = false; running_ = true; parent_->suspended_ = true; diff --git a/src/kernel/rng.cpp b/src/kernel/rng.cpp index 00cec9c1d9..f40492d9ae 100644 --- a/src/kernel/rng.cpp +++ b/src/kernel/rng.cpp @@ -27,7 +27,7 @@ struct alignas(SMP_ALIGN) rng_state { uint64_t state[25]; }; -static SMP_ARRAY rng; +static SMP::Array rng; static inline uint64_t rotate_left(uint64_t input, size_t rot) { return (input << rot) | (input >> (64-rot)); diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index 3f98f06446..db58fcd7a2 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -89,7 +89,7 @@ struct alignas(SMP_ALIGN) context_buffer std::array buffer; bool is_panicking = false; }; -static SMP_ARRAY contexts; +static SMP::Array contexts; size_t get_crash_context_length() { diff --git a/src/kernel/timers.cpp b/src/kernel/timers.cpp index 653901880d..18297e857a 100644 --- a/src/kernel/timers.cpp +++ b/src/kernel/timers.cpp @@ -81,7 +81,7 @@ struct alignas(SMP_ALIGN) timer_system uint32_t* periodic_started = nullptr; uint32_t* periodic_stopped = nullptr; }; -static SMP_ARRAY systems; +static SMP::Array systems; void timer_system::free_timer(Timers::id_t id) { diff --git a/src/musl/gettid.cpp b/src/musl/gettid.cpp index 1e9cdebbc5..525c957a0a 100644 --- a/src/musl/gettid.cpp +++ b/src/musl/gettid.cpp @@ -1,14 +1,13 @@ -#include "common.hpp" +#include "stub.hpp" static long sys_gettid() { -#ifdef INCLUDEOS_SINGLE_THREADED - return 0; -#else -#error "gettid not implemented for threaded IncludeOS" +#ifndef INCLUDEOS_SINGLE_THREADED +#warning "gettid not implemented for threaded IncludeOS" #endif + return 1; } extern "C" long syscall_SYS_gettid() { - return strace(sys_gettid, "gettid"); + return stubtrace(sys_gettid, "gettid"); } diff --git a/src/musl/mmap.diff b/src/musl/mmap.diff new file mode 100644 index 0000000000..558adef373 --- /dev/null +++ b/src/musl/mmap.diff @@ -0,0 +1,79 @@ +diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp +index 511b92a..a6c836c 100644 +--- a/src/musl/mmap.cpp ++++ b/src/musl/mmap.cpp +@@ -1,19 +1,71 @@ + #include "common.hpp" + #include + +#include + +#include + +#include + +#include + + extern uintptr_t heap_begin; + +uintptr_t heap_end = 0; + static uintptr_t current_pos = 0; + + + +using Alloc = util::alloc::Lstack<4096>; + +static Alloc alloc; + + + +void init_mmap(uintptr_t addr_begin){ + + if (alloc.empty()){ + + printf("Alloc not empty: %s\n ", alloc.begin() == nullptr ? "nullptr" : "size not 0"); + + } + + Expects(alloc.empty()); + + auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); + + alloc.donate((void*)aligned_begin, (OS::heap_max() - aligned_begin) & ~(Alloc::align - 1)); + + + +} + + + +void* kalloc(size_t size){ + + return alloc.allocate(size); + +} + + + +void kfree (void* ptr, size_t size){ + + alloc.deallocate(ptr, size); + +} + + + extern "C" + void* syscall_SYS_mmap(void *addr, size_t length, int prot, int flags, + int fd, off_t offset) + { + - uintptr_t res = heap_begin + current_pos; + - current_pos += length; + + + + STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d \n", + + addr, length, prot, flags, fd, offset); + + + + // TODO: Mapping to file descriptor + + if (fd > 0) { + + assert(false && "Mapping to file descriptor not yet implemented"); + + } + + + + // TODO: mapping virtual address + + if (addr) { + + errno = ENODEV; + + return MAP_FAILED; + + } + + + + void* res = kalloc(length); + STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d res=%p\n", + addr, length, prot, flags, fd, offset, res); + - return (void*)res; + + + + return res; + } + ++ ++/** ++ The mmap2() system call provides the same interface as mmap(2), ++ except that the final argument specifies the offset into the file in ++ 4096-byte units (instead of bytes, as is done by mmap(2)). This ++ enables applications that use a 32-bit off_t to map large files (up ++ to 2^44 bytes). ++ ++ http://man7.org/linux/man-pages/man2/mmap2.2.html ++**/ + extern "C" + void* syscall_SYS_mmap2(void *addr, size_t length, int prot, + int flags, int fd, off_t offset) { diff --git a/src/musl/set_tid_address.cpp b/src/musl/set_tid_address.cpp index 64619decf5..206a4379aa 100644 --- a/src/musl/set_tid_address.cpp +++ b/src/musl/set_tid_address.cpp @@ -1,16 +1,12 @@ -#include "common.hpp" +#include "stub.hpp" #warning stub -#ifdef INCLUDEOS_SINGLE_THREADED -struct { +static struct { int tid = 1; int* set_child_tid = nullptr; int* clear_child_tid = nullptr; } __main_thread__; -#else -#warning syscall set_tid_address is not yet thread safe -#endif static long sys_set_tid_address(int* tidptr) { __main_thread__.clear_child_tid = tidptr; @@ -19,5 +15,5 @@ static long sys_set_tid_address(int* tidptr) { extern "C" long syscall_SYS_set_tid_address(int* tidptr) { - return strace(sys_set_tid_address, "set_tid_address", tidptr); + return stubtrace(sys_set_tid_address, "set_tid_address", tidptr); } diff --git a/src/musl/tkill.cpp b/src/musl/tkill.cpp index ba385431b0..3a20927cfe 100644 --- a/src/musl/tkill.cpp +++ b/src/musl/tkill.cpp @@ -1,15 +1,16 @@ -#include "common.hpp" +#include "stub.hpp" #include static int sys_tkill(int tid, int sig) { - #ifdef INCLUDEOS_SINGLE_THREADED + +#ifndef INCLUDEOS_SINGLE_THREADED +#warning "tkill not implemented for threaded IncludeOS" +#endif + exit(sig); - #else - #error "tkill not implemented for threaded IncludeOS" - #endif } extern "C" int syscall_SYS_tkill(int tid, int sig) { - return strace(sys_tkill, "tkill", tid, sig); + return stubtrace(sys_tkill, "tkill", tid, sig); } diff --git a/src/platform/kvm/kvmclock.cpp b/src/platform/kvm/kvmclock.cpp index f6dfe2d637..61057dc573 100644 --- a/src/platform/kvm/kvmclock.cpp +++ b/src/platform/kvm/kvmclock.cpp @@ -21,7 +21,7 @@ struct alignas(4096) pvclock_vcpu_time_info { unsigned char flags; unsigned char pad[2]; }__attribute__((packed)); -static SMP_ARRAY vcpu_time; +static SMP::Array vcpu_time; struct alignas(4096) pvclock_wall_clock { uint32_t version; diff --git a/src/platform/x86_pc/apic_revenant.cpp b/src/platform/x86_pc/apic_revenant.cpp index b27704db8b..72fb7bf353 100644 --- a/src/platform/x86_pc/apic_revenant.cpp +++ b/src/platform/x86_pc/apic_revenant.cpp @@ -1,5 +1,4 @@ #include "apic_revenant.hpp" - #include "apic.hpp" #include "apic_timer.hpp" #include "clocks.hpp" @@ -9,13 +8,17 @@ #include #include +namespace x86 { + extern void initialize_cpu_tables_for_cpu(int); + smp_stuff smp_main; + SMP::Array smp_system; +} + extern "C" void* get_cpu_esp(); extern "C" void lapic_exception_handler(); #define INFO(FROM, TEXT, ...) printf("%13s ] " TEXT "\n", "[ " FROM, ##__VA_ARGS__) using namespace x86; -smp_stuff smp_main; -SMP_ARRAY smp_system; static bool revenant_task_doer(smp_system_stuff& system) { @@ -76,7 +79,7 @@ void revenant_main(int cpu) // enable Local APIC x86::APIC::get().smp_enable(); // setup GDT & per-cpu feature - initialize_gdt_for_cpu(cpu); + initialize_cpu_tables_for_cpu(cpu); #ifdef ARCH_x86_64 // interrupt stack tables ist_initialize_for_cpu(cpu, this_stack); @@ -109,6 +112,9 @@ void revenant_main(int cpu) // signal that the revenant has started smp_main.boot_barrier.inc(); + SMP::global_lock(); + x86::smp_main.initialized_cpus.push_back(cpu); + SMP::global_unlock(); while (true) { Events::get().process_events(); diff --git a/src/platform/x86_pc/apic_revenant.hpp b/src/platform/x86_pc/apic_revenant.hpp index a91020ec17..bee479ee41 100644 --- a/src/platform/x86_pc/apic_revenant.hpp +++ b/src/platform/x86_pc/apic_revenant.hpp @@ -23,9 +23,11 @@ #include #include #include +#include extern "C" void revenant_main(int); +namespace x86 { struct smp_task { smp_task(SMP::task_func a, SMP::done_func b) @@ -40,10 +42,12 @@ struct smp_stuff uintptr_t stack_base; uintptr_t stack_size; minimal_barrier_t boot_barrier; - uint32_t bmp_storage[1] = {0}; + std::vector initialized_cpus {0}; MemBitmap bitmap{&bmp_storage[0], 1}; }; + + extern smp_stuff smp_main; struct smp_system_stuff @@ -54,6 +58,7 @@ struct smp_system_stuff std::vector completed; bool work_done; }; -extern SMP_ARRAY smp_system; + extern SMP::Array smp_system; +} #endif diff --git a/src/platform/x86_pc/clocks.cpp b/src/platform/x86_pc/clocks.cpp index c47429c2cf..03ae909318 100644 --- a/src/platform/x86_pc/clocks.cpp +++ b/src/platform/x86_pc/clocks.cpp @@ -36,7 +36,7 @@ struct sysclock_t wall_time_t wall_time = nullptr; tsc_khz_t tsc_khz = nullptr; }; -static SMP_ARRAY vcpu_clock; +static SMP::Array vcpu_clock; namespace x86 { diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 13043fe479..744de9602e 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -50,7 +50,7 @@ extern uintptr_t _ELF_END_; struct alignas(SMP_ALIGN) OS_CPU { uint64_t cycles_hlt = 0; }; -static SMP_ARRAY os_per_cpu; +static SMP::Array os_per_cpu; uint64_t OS::cycles_asleep() noexcept { return PER_CPU(os_per_cpu).cycles_hlt; diff --git a/src/platform/x86_pc/platform.cpp b/src/platform/x86_pc/platform.cpp index ca1ce4b06a..9f877ba72c 100644 --- a/src/platform/x86_pc/platform.cpp +++ b/src/platform/x86_pc/platform.cpp @@ -40,8 +40,12 @@ struct alignas(64) smp_table int cpuid; /** put more here **/ }; +static SMP::Array cpu_tables; + +namespace x86 { + void initialize_cpu_tables_for_cpu(int cpu); +} -static SMP_ARRAY cpu_tables; void __platform_init() { @@ -59,19 +63,8 @@ void __platform_init() x86::ist_initialize_for_cpu(0, 0x9D3F0); #endif - // IDT manager: Interrupt and exception handlers - //INFO("x86", "Creating CPU exception handlers"); - // x86::idt_initialize_for_cpu(0); - - cpu_tables[0].cpuid = 0; - -#ifdef ARCH_x86_64 - x86::CPU::set_gs(&cpu_tables[0]); -#else - x86::CPU::set_fs(&cpu_tables[0]); -#endif - - MYINFO("Creating CPU exception handlers"); + INFO("x86", "Initializing CPU 0"); + x86::initialize_cpu_tables_for_cpu(0); Events::get(0).init_local(); // setup APIC, APIC timer, SMP etc. @@ -111,6 +104,16 @@ void __platform_init() kprintf("Platform init done \n"); } +void x86::initialize_cpu_tables_for_cpu(int cpu){ + cpu_tables[cpu].cpuid = cpu; + +#ifdef ARCH_x86_64 + x86::CPU::set_gs(&cpu_tables[cpu]); +#else + x86::CPU::set_fs(&cpu_tables[cpu]); +#endif +} + void __arch_enable_legacy_irq(uint8_t irq) { x86::APIC::enable_irq(irq); diff --git a/src/platform/x86_pc/smp.cpp b/src/platform/x86_pc/smp.cpp index ed2e35ee21..5e94bb86a8 100644 --- a/src/platform/x86_pc/smp.cpp +++ b/src/platform/x86_pc/smp.cpp @@ -137,6 +137,8 @@ void init_SMP() } // x86 +using namespace x86; + /// implementation of the SMP interface /// int SMP::cpu_id() noexcept { @@ -152,11 +154,11 @@ int SMP::cpu_id() noexcept } int SMP::cpu_count() noexcept { -#ifdef INCLUDEOS_SINGLE_THREADED - return 1; -#else - return x86::ACPI::get_cpus().size(); -#endif + return x86::smp_main.initialized_cpus.size(); +} + +const std::vector& SMP::active_cpus(){ + return x86::smp_main.initialized_cpus; } __attribute__((weak)) @@ -240,52 +242,3 @@ void SMP::global_unlock() noexcept { unlock(__global_lock); } - -/// SMP variants of malloc and free /// -#ifndef INCLUDEOS_SINGLE_THREADED -static spinlock_t __memory_lock = 0; - -#include -void* malloc(size_t size) -{ - lock(__memory_lock); - void* addr = _malloc_r(_REENT, size); - unlock(__memory_lock); - return addr; -} -void* calloc(size_t num, size_t size) -{ - lock(__memory_lock); - void* addr = _calloc_r(_REENT, num, size); - unlock(__memory_lock); - return addr; -} -void* realloc(void *ptr, size_t new_size) -{ - lock(__memory_lock); - void* addr = _realloc_r (_REENT, ptr, new_size); - unlock(__memory_lock); - return addr; -} -void free(void* ptr) -{ - lock(__memory_lock); - _free_r(_REENT, ptr); - unlock(__memory_lock); -} -void* posix_memalign(size_t align, size_t nbytes) -{ - lock(__memory_lock); - void* addr = _memalign_r(_REENT, align, nbytes); - unlock(__memory_lock); - return addr; -} -void* memalign(size_t align, size_t nbytes) -{ - lock(__memory_lock); - void* addr = _memalign_r(_REENT, align, nbytes); - unlock(__memory_lock); - return addr; -} - -#endif diff --git a/test/kernel/integration/fiber/fiber_smp.cpp b/test/kernel/integration/fiber/fiber_smp.cpp index dc27ea307f..e62981eebb 100644 --- a/test/kernel/integration/fiber/fiber_smp.cpp +++ b/test/kernel/integration/fiber/fiber_smp.cpp @@ -269,15 +269,16 @@ void many_fibers() { for (int i = 1; i < SMP::cpu_count(); ++i) { active_runners++; - SMP::add_task([i]{ + int cpu = SMP::active_cpus(i); + SMP::add_task([cpu]{ - CPULOG("[ SMP::add_task %i ] starting \n", i); + CPULOG("[ SMP::add_task %i ] starting \n", cpu); - Fiber runner { threadrunner , i}; + Fiber runner { threadrunner , cpu}; auto runner_id_ = runner.id(); runners_per_cpu[SMP::cpu_id()] = runner_id_; CPULOG("[ SMP::add_task %i ] NEW RUNNER: id %i, th_cnt: %i\n", - i, runner.id(), active_runners.load()); + cpu, runner.id(), active_runners.load()); runner.start(); Expects(runner.id() == runner_id_); active_runners--; @@ -285,7 +286,7 @@ void many_fibers() { asm("hlt"); CPULOG("[ SMP::add_task %i ] runner id %i done. Runners left: %i\n", i, runner.id(), active_runners.load());*/ - }, i); + }, cpu); } diff --git a/test/kernel/integration/fiber/service.cpp b/test/kernel/integration/fiber/service.cpp index 18af8b726d..a3aca32477 100644 --- a/test/kernel/integration/fiber/service.cpp +++ b/test/kernel/integration/fiber/service.cpp @@ -312,8 +312,12 @@ void Service::start() #ifndef INCLUDEOS_SINGLE_THREADED - extern void fiber_smp_test(); - fiber_smp_test(); + if (SMP::cpu_count() > 1) { + extern void fiber_smp_test(); + fiber_smp_test(); + } else { + INFO("Service", "SMP test requires > 1 cpu's, found %i \n", SMP::cpu_count()); + } #endif SMP_PRINT("Service done. rsp @ %p \n", get_rsp()); diff --git a/test/kernel/integration/smp/service.cpp b/test/kernel/integration/smp/service.cpp index e882b4b5a3..0e33319224 100644 --- a/test/kernel/integration/smp/service.cpp +++ b/test/kernel/integration/smp/service.cpp @@ -28,7 +28,7 @@ struct alignas(SMP_ALIGN) per_cpu_test int value; }; -static SMP_ARRAY testing; +static SMP::Array testing; #include void smp_advanced_test() @@ -80,7 +80,7 @@ void smp_advanced_test() [] (int) { static int times = 0; SMP::global_lock(); - printf("This is timer from a CPU core\n"); + printf("This is timer from CPU core %d\n", SMP::cpu_id()); times++; if (times == SMP::cpu_count()-1 @@ -116,8 +116,24 @@ void SMP::init_task() void Service::start() { + + for (const auto& i : SMP::active_cpus()) + { + SMP::global_lock(); + printf("CPU %i active \n", i); + SMP::global_unlock(); + + SMP::add_task([i]{ + SMP::global_lock(); + printf("CPU %i, id %i running task \n", i, SMP::cpu_id()); + SMP::global_unlock(); + }, i); + + SMP::signal(i); + } // trigger interrupt SMP::broadcast(IRQ); + // the rest smp_advanced_test(); } From 9279c217bb90b1b258aad7642cd8447e42710488 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 19 Feb 2018 12:37:13 +0100 Subject: [PATCH 061/723] posix: Made file descriptor includes public --- {src/include => api/posix}/fd.hpp | 0 {src/include => api/posix}/fd_map.hpp | 0 {src/include => api/posix}/file_fd.hpp | 0 {src/include => api/posix}/sockfd.hpp | 0 {src/include => api/posix}/tcp_fd.hpp | 0 {src/include => api/posix}/udp_fd.hpp | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename {src/include => api/posix}/fd.hpp (100%) rename {src/include => api/posix}/fd_map.hpp (100%) rename {src/include => api/posix}/file_fd.hpp (100%) rename {src/include => api/posix}/sockfd.hpp (100%) rename {src/include => api/posix}/tcp_fd.hpp (100%) rename {src/include => api/posix}/udp_fd.hpp (100%) diff --git a/src/include/fd.hpp b/api/posix/fd.hpp similarity index 100% rename from src/include/fd.hpp rename to api/posix/fd.hpp diff --git a/src/include/fd_map.hpp b/api/posix/fd_map.hpp similarity index 100% rename from src/include/fd_map.hpp rename to api/posix/fd_map.hpp diff --git a/src/include/file_fd.hpp b/api/posix/file_fd.hpp similarity index 100% rename from src/include/file_fd.hpp rename to api/posix/file_fd.hpp diff --git a/src/include/sockfd.hpp b/api/posix/sockfd.hpp similarity index 100% rename from src/include/sockfd.hpp rename to api/posix/sockfd.hpp diff --git a/src/include/tcp_fd.hpp b/api/posix/tcp_fd.hpp similarity index 100% rename from src/include/tcp_fd.hpp rename to api/posix/tcp_fd.hpp diff --git a/src/include/udp_fd.hpp b/api/posix/udp_fd.hpp similarity index 100% rename from src/include/udp_fd.hpp rename to api/posix/udp_fd.hpp From d0d9666b1cca592d6ae3abe19655e9a84b1b44c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 19 Feb 2018 12:38:31 +0100 Subject: [PATCH 062/723] fs: Added FD compatible concept to VFS --- api/fs/fd_compatible.hpp | 38 +++++++++++++++++++++++++++++++++++ api/fs/vfs.hpp | 43 ++++++++++++++++++++++++++-------------- 2 files changed, 66 insertions(+), 15 deletions(-) create mode 100644 api/fs/fd_compatible.hpp diff --git a/api/fs/fd_compatible.hpp b/api/fs/fd_compatible.hpp new file mode 100644 index 0000000000..248c591b95 --- /dev/null +++ b/api/fs/fd_compatible.hpp @@ -0,0 +1,38 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef INCLUDE_FD_COMPATIBLE_HPP +#define INCLUDE_FD_COMPATIBLE_HPP + +#include +#include + +/** + * @brief Makes classes inheriting this carry the + * attribute to be able to create a + * file descriptor of the resource. + */ +class FD_compatible { +public: + delegate open_fd = nullptr; + + virtual ~FD_compatible() = default; + +}; + +#endif diff --git a/api/fs/vfs.hpp b/api/fs/vfs.hpp index 738b3281cd..cb22b0f5bb 100644 --- a/api/fs/vfs.hpp +++ b/api/fs/vfs.hpp @@ -27,6 +27,7 @@ #include #include #include +#include // Demangle extern "C" char* __cxa_demangle(const char* mangled_name, @@ -54,29 +55,32 @@ inline std::string type_name(const std::type_info& type_, size_t max_chars = 0) namespace fs { - /** Exception: Trying to fetch an object of wrong type **/ - struct Err_bad_cast : public std::runtime_error { + struct VFS_err : public std::runtime_error { using runtime_error::runtime_error; }; + /** Exception: Trying to fetch an object of wrong type **/ + struct Err_bad_cast : public VFS_err { + using VFS_err::VFS_err; + }; /** Exception: trying to fetch object from non-leaf **/ - struct Err_not_leaf : public std::runtime_error { - using runtime_error::runtime_error; + struct Err_not_leaf : public VFS_err { + using VFS_err::VFS_err; }; /** Exception: trying to access children of non-parent **/ - struct Err_not_parent : public std::runtime_error { - using runtime_error::runtime_error; + struct Err_not_parent : public VFS_err { + using VFS_err::VFS_err; }; /** Exception: trying to access non-existing node **/ - struct Err_not_found : public std::runtime_error { - using runtime_error::runtime_error; + struct Err_not_found : public VFS_err { + using VFS_err::VFS_err; }; /** Exception: trying to mount on an occupied or non-existing mount point **/ - struct Err_mountpoint_invalid : public std::runtime_error { - using runtime_error::runtime_error; + struct Err_mountpoint_invalid : public VFS_err { + using VFS_err::VFS_err; }; @@ -109,12 +113,16 @@ namespace fs { VFS_entry(T& obj, const std::string& name, const std::string& desc) // Can't have default desc due to clash with next ctor : type_{typeid(T)}, obj_is_const{std::is_const::value}, obj_{const_cast::type*>(&obj)}, - name_{name}, desc_{desc} - {} + name_{name}, desc_{desc}, + is_fd_compatible{std::is_base_of::value} + { + } VFS_entry(std::string name, std::string desc) - : type_{typeid(nullptr)}, obj_{nullptr}, name_{name}, desc_{desc} - {} + : type_{typeid(nullptr)}, obj_{nullptr}, name_{name}, desc_{desc}, + is_fd_compatible{false} + { + } VFS_entry() = delete; VFS_entry(VFS_entry&) = delete; @@ -133,8 +141,12 @@ namespace fs { if (UNLIKELY(not obj_)) throw Err_not_leaf(name_ + " does not hold an object "); - if (UNLIKELY(typeid(T) != type_)) + // if not the same type, and not FD_compatible + if (UNLIKELY(typeid(T) != type_ and + not (std::is_same::value and this->is_fd_compatible))) + { throw Err_bad_cast(name_ + " is not of type " + ::type_name(typeid(T))); + } if (UNLIKELY(obj_is_const and not std::is_const::value)) throw Err_bad_cast(name_ + " must be retrieved as const " + type_name()); @@ -289,6 +301,7 @@ namespace fs { std::string name_; std::string desc_; std::vector children_; + bool is_fd_compatible = false; }; // End VFS_entry From 23842bf79e2deac75aabeff18ea7f4ac1e1f29f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 19 Feb 2018 12:40:31 +0100 Subject: [PATCH 063/723] kernel: RNG now singelton and fd compatible --- api/kernel/rng.hpp | 20 ++++++++++++++++---- src/kernel/os.cpp | 2 +- src/platform/x86_pc/apic_revenant.cpp | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/api/kernel/rng.hpp b/api/kernel/rng.hpp index 0a10042601..d97ab1526b 100644 --- a/api/kernel/rng.hpp +++ b/api/kernel/rng.hpp @@ -1,7 +1,7 @@ // -*-C++-*- // This file is a part of the IncludeOS unikernel - www.includeos.org // -// Copyright 2016 Oslo and Akershus University College of Applied Sciences +// Copyright 2016-2018 Oslo and Akershus University College of Applied Sciences // and Alfred Bratterud // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -40,9 +40,21 @@ inline uint32_t rng_extract_uint32() return x; } -struct RNG -{ - static void init(); + +#include +class RNG : public FD_compatible { +public: + static RNG& get() + { + static RNG rng; + return rng; + } + + void init(); + +private: + RNG() {} + }; diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 9d337c248a..911f991f56 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -117,7 +117,7 @@ void OS::post_start() MYINFO("Initializing RNG"); PROFILE("RNG init"); - RNG::init(); + RNG::get().init(); // Seed rand with 32 bits from RNG srand(rng_extract_uint32()); diff --git a/src/platform/x86_pc/apic_revenant.cpp b/src/platform/x86_pc/apic_revenant.cpp index 72fb7bf353..406fdcb6e2 100644 --- a/src/platform/x86_pc/apic_revenant.cpp +++ b/src/platform/x86_pc/apic_revenant.cpp @@ -104,7 +104,7 @@ void revenant_main(int cpu) Events::get().subscribe(0, revenant_task_handler); Events::get().subscribe(1, APIC_Timer::start_timers); // seed RNG - RNG::init(); + RNG::get().init(); // allow programmers to do stuff on each core at init SMP::init_task(); From 98462699e111a1b741a37b3c9235e97af6c088d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 19 Feb 2018 12:43:31 +0100 Subject: [PATCH 064/723] build: Build FD, linking etc --- api/posix/file_fd.hpp | 3 +-- cmake/post.service.cmake | 1 + src/CMakeLists.txt | 2 ++ src/posix/fd.cpp | 11 +++++------ 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/api/posix/file_fd.hpp b/api/posix/file_fd.hpp index 9dce612c64..f7ea47311e 100644 --- a/api/posix/file_fd.hpp +++ b/api/posix/file_fd.hpp @@ -18,8 +18,7 @@ #ifndef FILE_FD_HPP #define FILE_FD_HPP -#include -#include +#include "fd.hpp" #include class File_FD : public FD { diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 1807dc2517..bc2c044060 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -459,6 +459,7 @@ target_link_libraries(service libcxx musl_syscalls + libos ${CRTN} ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 1edfea615c..afa4973edb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -59,6 +59,8 @@ set(OS_OBJECTS net/nat/nat.cpp net/nat/napt.cpp fs/disk.cpp fs/filesystem.cpp fs/dirent.cpp fs/mbr.cpp fs/path.cpp fs/fat.cpp fs/fat_async.cpp fs/fat_sync.cpp fs/memdisk.cpp + # POSIX + posix/fd.cpp ) add_library(os STATIC ${OS_OBJECTS}) diff --git a/src/posix/fd.cpp b/src/posix/fd.cpp index 1351f2a18d..c1cf812d70 100644 --- a/src/posix/fd.cpp +++ b/src/posix/fd.cpp @@ -15,15 +15,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include #include -#include int FD::fcntl(int cmd, va_list list) { - PRINT("fcntl(%d)\n", cmd); + //PRINT("fcntl(%d)\n", cmd); switch (cmd) { case F_GETFD: // return descriptor flags @@ -49,20 +48,20 @@ int FD::fcntl(int cmd, va_list list) int FD::ioctl(int req, void* arg) { - PRINT("ioctl(%d, %p) = -1\n", req, arg); + //PRINT("ioctl(%d, %p) = -1\n", req, arg); errno = ENOSYS; return -1; } int FD::getsockopt(int fd, int, void *__restrict__, socklen_t *__restrict__) { - PRINT("getsockopt(%d) = -1\n", fd); + //PRINT("getsockopt(%d) = -1\n", fd); errno = ENOTSOCK; return -1; } int FD::setsockopt(int fd, int, const void *, socklen_t) { - PRINT("setsockopt(%d) = -1\n", fd); + //PRINT("setsockopt(%d) = -1\n", fd); errno = ENOTSOCK; return -1; } From 810bad35299782910703ba44521809758605897e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 19 Feb 2018 12:45:07 +0100 Subject: [PATCH 065/723] plugin: Added VFS plugin (including RNG fd) --- api/posix/rng_fd.hpp | 51 ++++++++++++++++++++++ src/plugins/CMakeLists.txt | 4 ++ src/plugins/vfs.cpp | 47 ++++++++++++++++++++ test/kernel/integration/rng/CMakeLists.txt | 2 + 4 files changed, 104 insertions(+) create mode 100644 api/posix/rng_fd.hpp create mode 100644 src/plugins/vfs.cpp diff --git a/api/posix/rng_fd.hpp b/api/posix/rng_fd.hpp new file mode 100644 index 0000000000..5bc6fc6743 --- /dev/null +++ b/api/posix/rng_fd.hpp @@ -0,0 +1,51 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef INCLUDE_RNG_FD_HPP +#define INCLUDE_RNG_FD_HPP + +#include +#include "fd.hpp" + +/** + * @brief Simple stateless file descriptor + * with random device functionality. + */ +class RNG_fd : public FD { +public: + RNG_fd(int fd) + : FD{fd} {} + + int read(void* output, size_t bytes) override + { + rng_extract(output, bytes); + return bytes; + } + + int write(const void* input, size_t bytes) override + { + rng_absorb(input, bytes); + return bytes; + } + + int close() override + { return 0; } + +}; // < class RNG_fd + +#endif diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index f958ccf47f..b4c592c4c1 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -23,6 +23,9 @@ add_dependencies(terminal_liu PrecompiledLibraries) add_library(nacl STATIC nacl.cpp) add_dependencies(nacl PrecompiledLibraries) +add_library(vfs STATIC vfs.cpp) +add_dependencies(vfs PrecompiledLibraries) + # # Installation # @@ -36,4 +39,5 @@ install(TARGETS terminal terminal_liu nacl + vfs DESTINATION includeos/${ARCH}/plugins) diff --git a/src/plugins/vfs.cpp b/src/plugins/vfs.cpp new file mode 100644 index 0000000000..bb11cd242c --- /dev/null +++ b/src/plugins/vfs.cpp @@ -0,0 +1,47 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file contains a plugin for automatically mounting +// resources to the virtual filesystem (VFS) + +#include +#include + +#include +// Mount RNG functionality to system paths +static void mount_rng() +{ + RNG::get().open_fd = []()->FD& { + return FD_map::_open(); + }; + + fs::mount("/dev/random", RNG::get(), "Random device"); + fs::mount("/dev/urandom", RNG::get(), "Random device"); +} + +// Function being run by the OS for mounting resources +static void vfs_mount() +{ + // RNG + mount_rng(); +} + +#include +__attribute__((constructor)) +static void vfs_mount_plugin() { + OS::register_plugin(vfs_mount, "VFS Mounter"); +} diff --git a/test/kernel/integration/rng/CMakeLists.txt b/test/kernel/integration/rng/CMakeLists.txt index 97d270b522..571962fa47 100644 --- a/test/kernel/integration/rng/CMakeLists.txt +++ b/test/kernel/integration/rng/CMakeLists.txt @@ -24,5 +24,7 @@ set(SOURCES service.cpp # ...add more here ) +set(PLUGINS vfs) + # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) From 4fd20ac122b3fb6b22f0a8b4c5842632a4726c08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 19 Feb 2018 12:48:16 +0100 Subject: [PATCH 066/723] musl: Updated systemcalls to use FD --- src/musl/open.cpp | 15 +++++++++++++++ src/musl/read.cpp | 17 ++++++++++++++--- src/musl/write.cpp | 20 +++++++++++--------- 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/src/musl/open.cpp b/src/musl/open.cpp index 03817690b1..724f5f1f81 100644 --- a/src/musl/open.cpp +++ b/src/musl/open.cpp @@ -1,7 +1,22 @@ #include "stub.hpp" #include +#include + int sys_open(const char *pathname, int flags, mode_t mode = 0) { + try { + auto& entry = fs::VFS::get(pathname); + auto& fd = entry.open_fd(); + return fd.get_id(); + } + catch(const fs::VFS_err&) { + // VFS error for one of many reasons (not mounted, not fd compatible etc) + } + catch(const std::bad_function_call&) { + // Open fd delegate not set + } + + errno = ENOENT; return -1; } diff --git a/src/musl/read.cpp b/src/musl/read.cpp index dafb5fbf9b..6cfc19d833 100644 --- a/src/musl/read.cpp +++ b/src/musl/read.cpp @@ -1,7 +1,18 @@ #include "common.hpp" +#include + +static ssize_t sys_read(int fd, void* buf, size_t len) { + try { + auto& fildes = FD_map::_get(fd); + return fildes.read(buf, len); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; + } +} extern "C" -long syscall_SYS_read() { - STUB("read"); - return 0; +ssize_t syscall_SYS_read(int fd, void *buf, size_t nbyte) { + return strace(sys_read, "read", fd, buf, nbyte); } diff --git a/src/musl/write.cpp b/src/musl/write.cpp index 602f3171ef..7922885c59 100644 --- a/src/musl/write.cpp +++ b/src/musl/write.cpp @@ -1,22 +1,24 @@ #include #include "common.hpp" +#include // The actual syscall static long sys_write(int fd, char* str, size_t len) { - if (fd <= 0) { - errno = EBADF; - return -1; - } - - if (fd == 1 or fd == 2) { + if (fd == 1 or fd == 2) + { OS::print(str, len); return len; } - // TODO: integrate with old file descriptors - errno = ENOSYS; - return -1; + try { + auto& fildes = FD_map::_get(fd); + return fildes.write(str, len); + } + catch(const FD_not_found&) { + errno = EBADF; + return -1; + } } // The syscall wrapper, using strace if enabled From b18ab0b9ab231314193017917b9e52fcd8e130dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 22 Feb 2018 10:37:10 +0100 Subject: [PATCH 067/723] fs: Make it possible to retrieve block size from a dirent --- api/fs/dirent.hpp | 3 +++ api/fs/fat.hpp | 3 +++ api/fs/filesystem.hpp | 3 +++ src/fs/disk.cpp | 2 +- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/api/fs/dirent.hpp b/api/fs/dirent.hpp index a3b308594c..04e06ae6ba 100644 --- a/api/fs/dirent.hpp +++ b/api/fs/dirent.hpp @@ -109,6 +109,9 @@ namespace fs { } //< switch (type) } + const File_system& fs() const noexcept + { return *fs_; } + /** Read async **/ inline void read(uint64_t pos, uint64_t n, on_read_func fn); diff --git a/api/fs/fat.hpp b/api/fs/fat.hpp index 7d2f54d8f0..cf1a887da9 100644 --- a/api/fs/fat.hpp +++ b/api/fs/fat.hpp @@ -70,6 +70,9 @@ namespace fs } return "Invalid fat type"; } + + uint64_t block_size() const noexcept override + { return device.block_size(); } /// ----------------------------------------------------- /// // constructor diff --git a/api/fs/filesystem.hpp b/api/fs/filesystem.hpp index ced156dd73..8577ea4c6f 100644 --- a/api/fs/filesystem.hpp +++ b/api/fs/filesystem.hpp @@ -71,6 +71,9 @@ namespace fs { /** Returns the name of this filesystem */ virtual std::string name() const = 0; + /** Returns the block size of this filesystem */ + virtual uint64_t block_size() const = 0; + /** Initialize this filesystem with LBA at @base_sector */ virtual void init(uint64_t lba, uint64_t size, on_init_func on_init) = 0; diff --git a/src/fs/disk.cpp b/src/fs/disk.cpp index e3592028a5..e5bf958a78 100644 --- a/src/fs/disk.cpp +++ b/src/fs/disk.cpp @@ -32,7 +32,7 @@ namespace fs { device.read( 0, hw::Block_device::on_read_func::make_packed( - [this, func] (hw::Block_device::buffer_t data) + [func] (hw::Block_device::buffer_t data) { std::vector parts; From 88e4132e1029a656dc37a1878a0c01d0617601e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 22 Feb 2018 15:30:29 +0100 Subject: [PATCH 068/723] musl: Added old stat implementation --- src/musl/stat.cpp | 55 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/src/musl/stat.cpp b/src/musl/stat.cpp index bb054b07ba..f9cc0f5a31 100644 --- a/src/musl/stat.cpp +++ b/src/musl/stat.cpp @@ -1,9 +1,58 @@ #include "common.hpp" +#include +#include +#include + +#include + +static inline unsigned round_up(unsigned n, unsigned div) { + Expects(div > 0); + return (n + div - 1) / div; +} + +static int sys_stat(const char *path, struct stat *buf) { + if (buf == nullptr) + { + //PRINT("stat(%s, %p) == %d\n", path, buf, -1); + errno = EFAULT; + return -1; + } + memset(buf, 0, sizeof(struct stat)); + try { + auto ent = fs::VFS::stat_sync(path); + if (ent.is_valid()) + { + if (ent.is_file()) buf->st_mode = S_IFREG; + if (ent.is_dir()) buf->st_mode = S_IFDIR; + buf->st_dev = ent.device_id(); + buf->st_ino = ent.block(); + buf->st_nlink = 1; + buf->st_size = ent.size(); + buf->st_atime = ent.modified(); + buf->st_ctime = ent.modified(); + buf->st_mtime = ent.modified(); + buf->st_blocks = buf->st_size > 0 ? round_up(buf->st_size, 512) : 0; + buf->st_blksize = ent.fs().block_size(); + //PRINT("stat(%s, %p) == %d\n", path, buf, 0); + return 0; + } + else { + //PRINT("stat(%s, %p) == %d\n", path, buf, -1); + errno = EIO; + return -1; + } + } + catch (...) + { + //PRINT("stat(%s, %p) == %d\n", path, buf, -1); + errno = EIO; + return -1; + } +} extern "C" -long syscall_SYS_stat() { - STUB("stat"); - return 0; +int syscall_SYS_stat(const char *pathname, struct stat *statbuf) { + return strace(sys_stat, "stat", pathname, statbuf); } extern "C" From 6a07a04a22f668ce1552a85d09562ece4bbb129c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 8 Mar 2018 10:09:22 +0100 Subject: [PATCH 069/723] musl: More syscalls related to files (stat) WIP --- api/posix/fd.hpp | 1 + cmake/post.service.cmake | 1 + src/musl/CMakeLists.txt | 17 ++++++++- src/musl/chmod.cpp | 13 +++++++ src/musl/clock_gettime.cpp | 2 +- src/musl/common.hpp | 6 +-- src/musl/creat.cpp | 12 ++++++ src/musl/cwd.cpp | 77 ++++++++++++++++++++++++++++++++++++++ src/musl/fchmod.cpp | 13 +++++++ src/musl/fchmodat.cpp | 14 +++++++ src/musl/fstat.cpp | 24 ++++++++++++ src/musl/fstatat.cpp | 35 +++++++++++++++++ src/musl/getdents.cpp | 12 ++++++ src/musl/mkdir.cpp | 12 ++++++ src/musl/mkdirat.cpp | 13 +++++++ src/musl/mknod.cpp | 14 +++---- src/musl/mknodat.cpp | 11 ++++++ src/musl/open.cpp | 25 ++++--------- src/musl/openat.cpp | 13 +++++++ src/musl/stat.cpp | 52 ++++++++++--------------- src/musl/umask.cpp | 18 +++++++++ src/musl/utimensat.cpp | 45 ++++++++++++++++++++++ src/musl/writev.cpp | 1 + 23 files changed, 366 insertions(+), 65 deletions(-) create mode 100644 src/musl/chmod.cpp create mode 100644 src/musl/creat.cpp create mode 100644 src/musl/cwd.cpp create mode 100644 src/musl/fchmod.cpp create mode 100644 src/musl/fchmodat.cpp create mode 100644 src/musl/fstat.cpp create mode 100644 src/musl/fstatat.cpp create mode 100644 src/musl/getdents.cpp create mode 100644 src/musl/mkdir.cpp create mode 100644 src/musl/mkdirat.cpp create mode 100644 src/musl/mknodat.cpp create mode 100644 src/musl/openat.cpp create mode 100644 src/musl/umask.cpp create mode 100644 src/musl/utimensat.cpp diff --git a/api/posix/fd.hpp b/api/posix/fd.hpp index 03991ee9ea..deb838ed8f 100644 --- a/api/posix/fd.hpp +++ b/api/posix/fd.hpp @@ -62,6 +62,7 @@ class FD { // file-related virtual int fchmod(mode_t) { return -1; } virtual int fchmodat(const char *, mode_t, int) { return -1; } + virtual long fstat(struct stat *) { return -1; } virtual int fstatat(const char *, struct stat *, int) { return -1; } virtual int futimens(const struct timespec[2]) { return -1; } virtual int utimensat(const char *, const struct timespec[2], int) { return -1; } diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index bc2c044060..aa315f7c99 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -435,6 +435,7 @@ target_link_libraries(service libarch libos libc + musl_syscalls cxxabi libpthread libunwind diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index ee8b4cd97e..094d6a01da 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -4,10 +4,23 @@ set(MUSL_OBJECTS write.cpp brk.cpp madvise.cpp mmap.cpp mremap.cpp munmap.cpp lseek.cpp sched_getaffinity.cpp sysinfo.cpp prlimit64.cpp getrlimit.cpp sched_yield.cpp set_robust_list.cpp - nanosleep.cpp open.cpp clock_gettime.cpp gettimeofday.cpp + nanosleep.cpp open.cpp creat.cpp clock_gettime.cpp gettimeofday.cpp poll.cpp exit_group.cpp exit.cpp close.cpp set_tid_address.cpp - pipe.cpp read.cpp readv.cpp getpid.cpp mknod.cpp stat.cpp sync.cpp msync.cpp + pipe.cpp read.cpp readv.cpp getpid.cpp mknod.cpp sync.cpp msync.cpp mincore.cpp syscall_n.cpp sigmask.cpp gettid.cpp tkill.cpp + stat.cpp fstat.cpp fstatat.cpp + chmod.cpp + cwd.cpp + fchmod.cpp + fchmodat.cpp + getdents.cpp + mkdir.cpp + mkdirat.cpp + mknodat.cpp + openat.cpp + umask.cpp + utimensat.cpp + ) add_library(musl_syscalls STATIC ${MUSL_OBJECTS}) diff --git a/src/musl/chmod.cpp b/src/musl/chmod.cpp new file mode 100644 index 0000000000..a794681461 --- /dev/null +++ b/src/musl/chmod.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" +#include + +static int sys_chmod(const char *path, mode_t mode) { + // currently makes no sense, especially since we're read-only + errno = EROFS; + return -1; +} + +extern "C" +int syscall_SYS_chmod(const char *path, mode_t mode) { + return strace(sys_chmod, "chmod", path, mode); +} diff --git a/src/musl/clock_gettime.cpp b/src/musl/clock_gettime.cpp index 63238a3ed4..29e0544bb7 100644 --- a/src/musl/clock_gettime.cpp +++ b/src/musl/clock_gettime.cpp @@ -2,6 +2,6 @@ extern "C" long syscall_SYS_clock_gettime() { - STUB("clock_gettime"); + //STUB("clock_gettime"); return 0; } diff --git a/src/musl/common.hpp b/src/musl/common.hpp index 6431bd4959..7af2d70f63 100644 --- a/src/musl/common.hpp +++ b/src/musl/common.hpp @@ -14,7 +14,7 @@ constexpr bool __strace = ENABLE_STRACE; extern "C" void __serial_print(const char*, size_t); template -inline constexpr void pr_param(std::ostream& out){ +inline constexpr void pr_param([[maybe_unused]] std::ostream& out){ } @@ -41,8 +41,8 @@ inline void strace_print(const char* name, Ret ret, Args&&... args){ out << name << "("; pr_param(out, args...); out << ") = " << ret; - if (errno) - out << " " << strerror(errno); + //if (errno) + // out << " " << strerror(errno); out << '\n'; auto str = out.str(); __serial_print(str.data(), str.size()); diff --git a/src/musl/creat.cpp b/src/musl/creat.cpp new file mode 100644 index 0000000000..3785b0adb1 --- /dev/null +++ b/src/musl/creat.cpp @@ -0,0 +1,12 @@ +#include "stub.hpp" +#include + +static long sys_creat(const char *pathname, mode_t mode) { + // currently makes no sense, especially since we're read-only + return -EROFS; +} + +extern "C" +long syscall_SYS_creat(const char *pathname, mode_t mode) { + return stubtrace(sys_creat, "creat", pathname, mode); +} diff --git a/src/musl/cwd.cpp b/src/musl/cwd.cpp new file mode 100644 index 0000000000..d3f945a1e9 --- /dev/null +++ b/src/musl/cwd.cpp @@ -0,0 +1,77 @@ +#include "common.hpp" +#include + +#include +#include +static std::string cwd{"/"}; + +static long sys_chdir(const char* path) +{ + // todo: handle relative path + // todo: handle .. + if (UNLIKELY(not path or strlen(path) < 1)) + return -ENOENT; + + if (strcmp(path, ".") == 0) + { + return 0; + } + std::string desired_path; + if (*path != '/') + { + desired_path = cwd; + if (!(desired_path.back() == '/')) desired_path += "/"; + desired_path += path; + } + else + { + desired_path.assign(path); + } + try { + auto ent = fs::VFS::stat_sync(desired_path); + if (ent.is_dir()) + { + cwd = desired_path; + assert(cwd.front() == '/'); + assert(cwd.find("..") == std::string::npos); + printf("cwd: %s\n", cwd.c_str()); + return 0; + } + else + { + // path is not a dir + return -ENOTDIR; + } + } + catch (const fs::Err_not_found& e) { + return -ENOTDIR; + } +} + +long sys_getcwd(char *buf, size_t size) +{ + Expects(cwd.front() == '/'); // dangerous? + + if (UNLIKELY(buf == nullptr or size == 0)) + return -EINVAL; + + if ((cwd.length() + 1) < size) + { + snprintf(buf, cwd.length()+1, "%s", cwd.c_str()); + return cwd.length()+1; + } + else + { + return -ERANGE; + } +} + +extern "C" +long syscall_SYS_chdir(const char* path) { + return strace(sys_chdir, "chdir", path); +} + +extern "C" +long syscall_SYS_getcwd(char *buf, size_t size) { + return strace(sys_getcwd, "getcwd", buf, size); +} diff --git a/src/musl/fchmod.cpp b/src/musl/fchmod.cpp new file mode 100644 index 0000000000..5ace900453 --- /dev/null +++ b/src/musl/fchmod.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" +#include + +static int sys_fchmod(int fildes, mode_t mode) { + // currently makes no sense, especially since we're read-only + errno = EROFS; + return -EROFS; +} + +extern "C" +int syscall_SYS_fchmod(int fildes, mode_t mode) { + return strace(sys_fchmod, "fchmod", fildes, mode); +} diff --git a/src/musl/fchmodat.cpp b/src/musl/fchmodat.cpp new file mode 100644 index 0000000000..881ed21506 --- /dev/null +++ b/src/musl/fchmodat.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include +#include + +static int sys_fchmodat(int fd, const char *path, mode_t mode, int flag) { + // currently makes no sense, especially since we're read-only + errno = EROFS; + return -EROFS; +} + +extern "C" +int syscall_SYS_fchmodat(int fd, const char *path, mode_t mode, int flag) { + return strace(sys_fchmodat, "fchmodat", fd, path, mode, flag); +} diff --git a/src/musl/fstat.cpp b/src/musl/fstat.cpp new file mode 100644 index 0000000000..d1523cfd6e --- /dev/null +++ b/src/musl/fstat.cpp @@ -0,0 +1,24 @@ +#include "common.hpp" +#include + +#include + +static long sys_fstat(int filedes, struct stat* stat_buf) +{ + if (UNLIKELY(stat_buf == nullptr)) + return -EINVAL; + + try { + auto& fd = FD_map::_get(filedes); + return fd.fstat(stat_buf); + } + catch(const FD_not_found&) { + return -EBADF; + } +} + +extern "C" +long syscall_SYS_fstat(int fd, struct stat* stat_buf) { + return strace(sys_fstat, "fstat", fd, stat_buf); +} + diff --git a/src/musl/fstatat.cpp b/src/musl/fstatat.cpp new file mode 100644 index 0000000000..9c6a1324e7 --- /dev/null +++ b/src/musl/fstatat.cpp @@ -0,0 +1,35 @@ +#include "common.hpp" +#include + +#include + +long sys_getcwd(char *buf, size_t size); +long sys_stat(const char *path, struct stat *buf); + +static long sys_fstatat(int filedes, const char *path, struct stat *buf, int flag) +{ + if (filedes == AT_FDCWD) + { + char cwd_buf[PATH_MAX]; + char abs_path[PATH_MAX]; + if (sys_getcwd(cwd_buf, PATH_MAX) > 0) { + snprintf(abs_path, PATH_MAX, "%s/%s", cwd_buf, path); + } + return sys_stat(abs_path, buf); + } + else + { + try { + auto& fd = FD_map::_get(filedes); + return fd.fstatat(path, buf, flag); + } + catch(const FD_not_found&) { + return -EBADF; + } + } +} + +extern "C" +long syscall_SYS_fstatat(int fd, const char *path, struct stat *stat_buf, int flag) { + return strace(sys_fstatat, "fstatat", fd, path, stat_buf, flag); +} diff --git a/src/musl/getdents.cpp b/src/musl/getdents.cpp new file mode 100644 index 0000000000..e8817916c0 --- /dev/null +++ b/src/musl/getdents.cpp @@ -0,0 +1,12 @@ +#include "common.hpp" + +static int sys_getdents(unsigned int fd, struct linux_dirent *dirp, unsigned int count) { + // currently makes no sense, especially since we're read-only + errno = EROFS; + return -1; +} + +extern "C" +int syscall_SYS_getdents(unsigned int fd, struct linux_dirent *dirp, unsigned int count) { + return strace(sys_getdents, "getdents", fd, dirp, count); +} diff --git a/src/musl/mkdir.cpp b/src/musl/mkdir.cpp new file mode 100644 index 0000000000..e3e9b7f724 --- /dev/null +++ b/src/musl/mkdir.cpp @@ -0,0 +1,12 @@ +#include "common.hpp" +#include + +static long sys_mkdir(const char *path, mode_t mode) { + // currently makes no sense, especially since we're read-only + return -EROFS; +} + +extern "C" +long syscall_SYS_mkdir(const char *path, mode_t mode) { + return strace(sys_mkdir, "mkdir", path, mode); +} diff --git a/src/musl/mkdirat.cpp b/src/musl/mkdirat.cpp new file mode 100644 index 0000000000..b74278004f --- /dev/null +++ b/src/musl/mkdirat.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" +#include +#include + +static long sys_mkdirat(int fd, const char *path, mode_t mode) { + // currently makes no sense, especially since we're read-only + return -EROFS; +} + +extern "C" +long syscall_SYS_mkdirat(int fd, const char *path, mode_t mode) { + return strace(sys_mkdirat, "mkdirat", fd, path, mode); +} diff --git a/src/musl/mknod.cpp b/src/musl/mknod.cpp index 1814890db7..255816113d 100644 --- a/src/musl/mknod.cpp +++ b/src/musl/mknod.cpp @@ -1,13 +1,11 @@ #include "common.hpp" -extern "C" { -long syscall_SYS_mknod() { - STUB("mknod"); - return 0; +static long sys_mknod(const char *pathname, mode_t mode, dev_t dev) { + // currently makes no sense, especially since we're read-only + return -EROFS; } -long syscall_SYS_mknodat() { - STUB("mknod"); - return 0; -} +extern "C" +long syscall_SYS_mknod(const char *pathname, mode_t mode, dev_t dev) { + return strace(sys_mknod, "mknod", pathname, mode, dev); } diff --git a/src/musl/mknodat.cpp b/src/musl/mknodat.cpp new file mode 100644 index 0000000000..43e4f5b562 --- /dev/null +++ b/src/musl/mknodat.cpp @@ -0,0 +1,11 @@ +#include "common.hpp" + +static long sys_mknodat(int dirfd, const char *pathname, mode_t mode, dev_t dev) { + // currently makes no sense, especially since we're read-only + return -EROFS; +} + +extern "C" +long syscall_SYS_mknodat(int dirfd, const char *pathname, mode_t mode, dev_t dev) { + return strace(sys_mknodat, "mknodat", dirfd, pathname, mode, dev); +} diff --git a/src/musl/open.cpp b/src/musl/open.cpp index 724f5f1f81..0b1b8254ef 100644 --- a/src/musl/open.cpp +++ b/src/musl/open.cpp @@ -1,9 +1,9 @@ -#include "stub.hpp" -#include +#include "common.hpp" +#include #include -int sys_open(const char *pathname, int flags, mode_t mode = 0) { +static long sys_open(const char *pathname, int flags, mode_t mode = 0) { try { auto& entry = fs::VFS::get(pathname); auto& fd = entry.open_fd(); @@ -16,21 +16,10 @@ int sys_open(const char *pathname, int flags, mode_t mode = 0) { // Open fd delegate not set } - errno = ENOENT; - return -1; + return -ENOENT; } -int sys_creat(const char *pathname, mode_t mode) { - return -1; +extern "C" +long syscall_SYS_open(const char *pathname, int flags, mode_t mode = 0) { + return strace(sys_open, "open", pathname, flags, mode); } - -extern "C" { -int syscall_SYS_open(const char *pathname, int flags, mode_t mode = 0) { - return stubtrace(sys_open, "open", pathname, flags, mode); -} - -int syscall_SYS_creat(const char *pathname, mode_t mode) { - return stubtrace(sys_creat, "creat", pathname, mode); -} - -} // extern "C" diff --git a/src/musl/openat.cpp b/src/musl/openat.cpp new file mode 100644 index 0000000000..f190ead14b --- /dev/null +++ b/src/musl/openat.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" +#include +#include + +static int sys_openat(int dirfd, const char *pathname, int flags, mode_t mode) { + errno = ENOSYS; + return -1; +} + +extern "C" +int syscall_SYS_openat(int dirfd, const char *pathname, int flags, mode_t mode) { + return strace(sys_openat, "openat", dirfd, pathname, flags, mode); +} diff --git a/src/musl/stat.cpp b/src/musl/stat.cpp index f9cc0f5a31..2f5cbab09d 100644 --- a/src/musl/stat.cpp +++ b/src/musl/stat.cpp @@ -1,22 +1,17 @@ #include "common.hpp" -#include #include -#include #include +#include // roundto -static inline unsigned round_up(unsigned n, unsigned div) { - Expects(div > 0); - return (n + div - 1) / div; -} +long sys_stat(const char *path, struct stat *buf) +{ + if (UNLIKELY(buf == nullptr)) + return -EFAULT; + + if (UNLIKELY(path == nullptr)) + return -ENOENT; -static int sys_stat(const char *path, struct stat *buf) { - if (buf == nullptr) - { - //PRINT("stat(%s, %p) == %d\n", path, buf, -1); - errno = EFAULT; - return -1; - } memset(buf, 0, sizeof(struct stat)); try { auto ent = fs::VFS::stat_sync(path); @@ -31,44 +26,35 @@ static int sys_stat(const char *path, struct stat *buf) { buf->st_atime = ent.modified(); buf->st_ctime = ent.modified(); buf->st_mtime = ent.modified(); - buf->st_blocks = buf->st_size > 0 ? round_up(buf->st_size, 512) : 0; + buf->st_blocks = buf->st_size > 0 ? util::bits::roundto<512>(buf->st_size) : 0; buf->st_blksize = ent.fs().block_size(); //PRINT("stat(%s, %p) == %d\n", path, buf, 0); return 0; } else { //PRINT("stat(%s, %p) == %d\n", path, buf, -1); - errno = EIO; - return -1; + return -EIO; } } catch (...) { //PRINT("stat(%s, %p) == %d\n", path, buf, -1); - errno = EIO; - return -1; + return -EIO; } } -extern "C" -int syscall_SYS_stat(const char *pathname, struct stat *statbuf) { - return strace(sys_stat, "stat", pathname, statbuf); -} - -extern "C" -long syscall_SYS_lstat() { - STUB("lstat"); - return 0; +static long sys_lstat(const char *path, struct stat *buf) +{ + // NOTE: should stat symlinks, instead of following them + return sys_stat(path, buf); } extern "C" -long syscall_SYS_fstat() { - STUB("fstat"); - return 0; +long syscall_SYS_stat(const char *path, struct stat *buf) { + return strace(sys_stat, "stat", path, buf); } extern "C" -long syscall_SYS_fstatat() { - STUB("fstatat"); - return 0; +long syscall_SYS_lstat(const char *path, struct stat *buf) { + return strace(sys_lstat, "lstat", path, buf); } diff --git a/src/musl/umask.cpp b/src/musl/umask.cpp new file mode 100644 index 0000000000..c22596949b --- /dev/null +++ b/src/musl/umask.cpp @@ -0,0 +1,18 @@ +#include "common.hpp" +#include + +mode_t THE_MASK = 002; + +static mode_t sys_umask(mode_t cmask) { + mode_t prev_mask = THE_MASK; + + if(THE_MASK != cmask) + THE_MASK = cmask; + + return prev_mask; +} + +extern "C" +mode_t syscall_SYS_umask(mode_t cmask) { + return strace(sys_umask, "umask", cmask); +} diff --git a/src/musl/utimensat.cpp b/src/musl/utimensat.cpp new file mode 100644 index 0000000000..d35b6a03c6 --- /dev/null +++ b/src/musl/utimensat.cpp @@ -0,0 +1,45 @@ +#include "common.hpp" +#include /* Definition of AT_* constants */ +#include +#include + +static int sys_utimensat(int dirfd, const char *pathname, + const struct timespec times[2], int flags) +{ + // currently makes no sense, especially since we're read-only + errno = EROFS; + return -1; +} + +// Obsolete, use utimensat +static int sys_futimesat(int dirfd, const char *pathname, + const struct timespec times[2]) +{ + return sys_utimensat(dirfd, pathname, times, 0); +} + +static int sys_utimes(const char *pathname, const struct timeval times[2]) +{ + // currently makes no sense, especially since we're read-only + errno = EROFS; + return -1; +} + +extern "C" +int syscall_SYS_utimensat(int dirfd, const char *pathname, + const struct timespec times[2], int flags) +{ + return strace(sys_utimensat, "utimensat", dirfd, pathname, times, flags); +} + +extern "C" +int syscall_SYS_futimesat(int dirfd, const char *pathname, + const struct timespec times[2]) +{ + return strace(sys_futimesat, "futimesat", dirfd, pathname, times); +} + +extern "C" +int syscall_SYS_utimes(const char *pathname, const struct timeval times[2]) { + return strace(sys_utimes, "utimes", pathname, times); +} diff --git a/src/musl/writev.cpp b/src/musl/writev.cpp index e81310ec8e..01f69bb6e8 100644 --- a/src/musl/writev.cpp +++ b/src/musl/writev.cpp @@ -1,5 +1,6 @@ #include "common.hpp" +#include #include static ssize_t sys_writev(int fd, const struct iovec *iov, int iovcnt) From 7c266437727a48d7a670fc052678a3b6b3bd22eb Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Mar 2018 14:08:18 +0100 Subject: [PATCH 070/723] test: more tests work with musl, add lest cmake option --- .gitignore | 1 + CMakeLists.txt | 8 +++- test/CMakeLists.txt | 8 ++-- test/kernel/integration/paging/service.cpp | 6 +-- .../integration/plugin_init/service.cpp | 38 +++++-------------- test/kernel/unit/x86_paging.cpp | 4 +- test/posix/unit/fd_map_test.cpp | 4 +- 7 files changed, 29 insertions(+), 40 deletions(-) diff --git a/.gitignore b/.gitignore index 47a3d1aa28..4c030ae675 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ *.img *.img.* *.gz +*.tar .DS_Store nbproject dummy.disk diff --git a/CMakeLists.txt b/CMakeLists.txt index 28da03c814..cda9342253 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,9 +201,15 @@ if(examples) add_subdirectory(examples) endif(examples) +option(lest "Install lest unittest headers" OFF) +if (lest) + init_submodule(test/lest) + install(DIRECTORY test/lest/include/lest DESTINATION ${CMAKE_INSTALL_PREFIX}/includeos/include) +endif() + option(tests "Build unit tests in /test and install lest test framework" OFF) if(tests) - init_submodule(test/lest) + set(lest ON) enable_testing() ExternalProject_Add(unittests PREFIX unittests diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index e8cf82c0ef..480c8d92a2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -128,6 +128,7 @@ set(TEST_SOURCES ) set(OS_SOURCES + ${SRC}/arch/x86_64/paging.cpp ${SRC}/fs/disk.cpp ${SRC}/fs/dirent.cpp ${SRC}/fs/fat.cpp @@ -153,7 +154,8 @@ set(OS_SOURCES ${SRC}/kernel/rng.cpp ${SRC}/kernel/service_stub.cpp ${SRC}/kernel/syscalls.cpp - ${SRC}/arch/x86_64/paging.cpp + ${SRC}/musl/mmap.cpp + ${SRC}/musl/brk.cpp ${SRC}/net/buffer_store.cpp ${SRC}/net/conntrack.cpp ${SRC}/net/dns/client.cpp @@ -190,6 +192,7 @@ set(OS_SOURCES ${SRC}/net/tcp/tcp.cpp ${SRC}/net/tcp/read_buffer.cpp ${SRC}/net/tcp/write_queue.cpp + ${SRC}/posix/fd.cpp ${SRC}/util/async.cpp ${SRC}/util/crc32.cpp ${SRC}/util/logger.cpp @@ -202,8 +205,6 @@ set(OS_SOURCES ${SRC}/util/tar.cpp ${SRC}/util/uri.cpp ${SRC}/virtio/virtio_queue.cpp - ${SRC}/musl/mmap.cpp - ${SRC}/musl/brk.cpp ) set(MOD_OBJECTS @@ -266,7 +267,6 @@ endif() add_executable(unittests ${SOURCES}) install(TARGETS unittests DESTINATION ${TEST}) -install(DIRECTORY lest/include/lest DESTINATION ${CMAKE_INSTALL_PREFIX}/include) if (GENERATE_SUPPORT_FILES) diff --git a/test/kernel/integration/paging/service.cpp b/test/kernel/integration/paging/service.cpp index 9b77ee15a9..2c4b932a57 100644 --- a/test/kernel/integration/paging/service.cpp +++ b/test/kernel/integration/paging/service.cpp @@ -21,9 +21,9 @@ #include #include #include -#include "../../../../api/kernel/memory.hpp" -#include "../../../../api/arch/x86_paging.hpp" -#include "../../../../api/arch/x86_paging_utils.hpp" +#include +#include +#include #include #include diff --git a/test/kernel/integration/plugin_init/service.cpp b/test/kernel/integration/plugin_init/service.cpp index a5759b377f..fbbca3e291 100644 --- a/test/kernel/integration/plugin_init/service.cpp +++ b/test/kernel/integration/plugin_init/service.cpp @@ -16,7 +16,6 @@ // limitations under the License. #include -#include extern int f1_data; extern int f2_data; @@ -25,37 +24,20 @@ extern int my_plugin_functions; extern bool example_plugin_registered; extern bool example_plugin_run; - -const lest::test specification[] = - { - { - CASE("Make sure the external example plugin was registered and called") - { - EXPECT(example_plugin_registered); - EXPECT(example_plugin_run); - } - }, - { - CASE( "Make sure the custom plugin initialization functions were called" ) - { - EXPECT(f1_data == 0xf1); - EXPECT(f2_data == 0xf2); - EXPECT(f3_data == 0xf3); - EXPECT(my_plugin_functions == 3); - } - } - }; - - - -void Service::start(const std::string&) +void Service::start() { INFO("Plugin test", "Testing the plugin initialization"); - auto failed = lest::run(specification, {"-p"}); - Expects(not failed); + CHECKSERT(example_plugin_registered, "Example plugin registered"); + CHECKSERT(example_plugin_run, "Example plugin ran"); - INFO("Plugin test", "SUCCESS"); + INFO("Plugin test", "Make sure the custom plugin initialization functions were called"); + CHECKSERT(f1_data == 0xf1, "f1 data OK"); + CHECKSERT(f2_data == 0xf2, "f2 data OK"); + CHECKSERT(f3_data == 0xf3, "f3 data OK"); + CHECKSERT(my_plugin_functions == 3, "Correct number of function calls"); + + INFO("Plugin test", "SUCCESS"); } diff --git a/test/kernel/unit/x86_paging.cpp b/test/kernel/unit/x86_paging.cpp index d822559cb5..7270929d24 100644 --- a/test/kernel/unit/x86_paging.cpp +++ b/test/kernel/unit/x86_paging.cpp @@ -19,8 +19,8 @@ #include #include -#include -#include +#include +#include #include #include diff --git a/test/posix/unit/fd_map_test.cpp b/test/posix/unit/fd_map_test.cpp index 201d342ea4..872b48695d 100644 --- a/test/posix/unit/fd_map_test.cpp +++ b/test/posix/unit/fd_map_test.cpp @@ -16,8 +16,8 @@ // limitations under the License. #include -#include -#include +#include +#include class Test_fd : public FD { public: From d2d3d5aac2edc7ceb0e041ecf6b3d994a4aee59c Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Mar 2018 14:08:41 +0100 Subject: [PATCH 071/723] protobuf: build for musl --- cmake/protobuf.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/protobuf.cmake b/cmake/protobuf.cmake index fb3659a9dc..612653e244 100644 --- a/cmake/protobuf.cmake +++ b/cmake/protobuf.cmake @@ -11,7 +11,7 @@ set(PROTOBUF_SRC ${LIB_PROTOBUF}/google/protobuf) include_directories(${LIB_PROTOBUF}) include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) -include_directories(${NEWLIB_INCLUDE_DIR}) +include_directories(${MUSL_INCLUDE_DIR}) # Maybe possible to use wildcard with files(...) to gather all cc objects. set(PROTOBUF_SOURCES From 48e2cf735e4dc0c9e91f8848e8e3e8d9a271af88 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Mar 2018 14:16:26 +0100 Subject: [PATCH 072/723] x86 musl: init elf parser --- src/arch/x86_64/linker.ld | 2 -- src/platform/x86_pc/kernel_start.cpp | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/linker.ld b/src/arch/x86_64/linker.ld index 2389d8b549..75d1c94da5 100644 --- a/src/arch/x86_64/linker.ld +++ b/src/arch/x86_64/linker.ld @@ -29,11 +29,9 @@ SECTIONS PROVIDE(_MULTIBOOT_START_ = .); *(.multiboot) } - PROVIDE(SECRET_SPACE_BEGIN = .) .text ALIGN(0x1000): { - PROVIDE(SECRET_SPACE_END = .) PROVIDE( _TEXT_START_ = . ); /* For solo5, although it's just used to print the mem layout. */ _stext = .; diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 8f17252c97..3f99bff5dd 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -38,6 +38,8 @@ extern "C" { uintptr_t _multiboot_free_begin(uintptr_t boot_addr); uintptr_t _move_symbols(uintptr_t loc); void _init_heap(uintptr_t free_mem_begin); + void _init_elf_parser(); + void _init_syscalls(); } extern char _ELF_START_; @@ -136,6 +138,12 @@ void kernel_start(uintptr_t magic, uintptr_t addr) kprintf("* Init heap\n"); _init_heap(free_mem_begin); + /// init ELF / backtrace functionality + _init_elf_parser(); + + // init syscalls + _init_syscalls(); + // Initialize CPU exceptions x86::idt_initialize_for_cpu(0); kprintf("* Thread local1: %i\n", __tl1__); From db919fbe67d97556e502f00b8ec7a15d0102b5a4 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Mar 2018 14:51:47 +0100 Subject: [PATCH 073/723] cmake: updated hash for new patched bundle --- cmake/cross_compiled_libraries.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cross_compiled_libraries.cmake b/cmake/cross_compiled_libraries.cmake index 5f4708c3fc..95d7020465 100644 --- a/cmake/cross_compiled_libraries.cmake +++ b/cmake/cross_compiled_libraries.cmake @@ -19,7 +19,7 @@ else(BUNDLE_LOC) ExternalProject_Add(PrecompiledLibraries PREFIX precompiled URL https://github.com/hioa-cs/IncludeOS/releases/download/v0.12.0-rc.2/IncludeOS_dependencies_v0-12-0_musl_libunwind_threaded.tar.gz - URL_HASH SHA1=4ab139536295321866ebe4136de25879450eb58b + URL_HASH SHA1=9f3f8da09c1f15bdc2a7d3feea665a31cb9fe889 CONFIGURE_COMMAND "" BUILD_COMMAND "" UPDATE_COMMAND "" From 50da88dee1efebec6ee5981469c21befa5718079 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Mar 2018 14:52:26 +0100 Subject: [PATCH 074/723] musl: patch endian.h to supress warnings --- etc/build_musl.sh | 1 + etc/musl/endian.patch | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 etc/musl/endian.patch diff --git a/etc/build_musl.sh b/etc/build_musl.sh index 1e0dd8236b..5f2eefc5e4 100755 --- a/etc/build_musl.sh +++ b/etc/build_musl.sh @@ -20,6 +20,7 @@ rm -f arch/i386/syscall_arch.h # Compatibility patch git apply $INCLUDEOS_SRC/etc/musl/musl.patch || true +git apply $INCLUDEOS_SRC/etc/musl/endian.patch || true # ln -s $INCLUDEOS_SRC/api/arch/syscalls.h arch/x86_64/syscall_arch.h diff --git a/etc/musl/endian.patch b/etc/musl/endian.patch new file mode 100644 index 0000000000..403af2cdc1 --- /dev/null +++ b/etc/musl/endian.patch @@ -0,0 +1,19 @@ +diff --git a/include/endian.h b/include/endian.h +index 1bd4445..88c3347 100644 +--- a/include/endian.h ++++ b/include/endian.h +@@ -29,12 +29,12 @@ static __inline uint16_t __bswap16(uint16_t __x) + + static __inline uint32_t __bswap32(uint32_t __x) + { +- return __x>>24 | __x>>8&0xff00 | __x<<8&0xff0000 | __x<<24; ++ return __x>>24 | (__x>>8&0xff00) | (__x<<8&0xff0000) | __x<<24; + } + + static __inline uint64_t __bswap64(uint64_t __x) + { +- return __bswap32(__x)+0ULL<<32 | __bswap32(__x>>32); ++ return (__bswap32(__x)+0ULL)<<32 | __bswap32(__x>>32); + } + + #if __BYTE_ORDER == __LITTLE_ENDIAN From 7f09b5ae54c54ec9cad9c6832883fad4c1dc8ecc Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Mar 2018 14:53:08 +0100 Subject: [PATCH 075/723] musl: remove warnings in kernel start --- src/platform/x86_pc/kernel_start.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 3f99bff5dd..5b9c9d4f89 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -28,7 +28,7 @@ #include "idt.hpp" #undef Expects -#define Expects(X) if (!X) { kprint("Expect failed: " #X "\n"); asm("cli;hlt"); } +#define Expects(X) if (!(X)) { kprint("Expect failed: " #X "\n"); asm("cli;hlt"); } extern "C" { void __init_serial1(); @@ -106,9 +106,9 @@ void kernel_start(uintptr_t magic, uintptr_t addr) __grub_addr = addr; // TODO: remove extra verbosity after musl port stabilizes - kprintf("\n////////////////// IncludeOS kernel start ////////////////// \n", + kprintf("\n////////////////// IncludeOS kernel start ////////////////// \n"); + kprintf("* Booted with magic 0x%lx, grub @ 0x%lx \n* Init sanity\n", magic, addr); - kprintf("* Booted with magic 0x%lx, grub @ 0x%lx \n* Init sanity\n", magic, addr); // generate checksums of read-only areas etc. __init_sanity_checks(); @@ -121,13 +121,13 @@ void kernel_start(uintptr_t magic, uintptr_t addr) if (magic == MULTIBOOT_BOOTLOADER_MAGIC) { free_mem_begin = _multiboot_free_begin(addr); - kprintf("* Free mem begin: %p \n", free_mem_begin); + kprintf("* Free mem begin: 0x%lx \n", free_mem_begin); } kprintf("* Moving symbols. \n"); // Preserve symbols from the ELF binary free_mem_begin += _move_symbols(free_mem_begin); - kprintf("* Free mem moved to: %p \n", free_mem_begin); + kprintf("* Free mem moved to: 0x%lx \n", free_mem_begin); kprintf("* Grub magic: 0x%lx, grub info @ 0x%lx\n", __grub_magic, __grub_addr); @@ -148,7 +148,7 @@ void kernel_start(uintptr_t magic, uintptr_t addr) x86::idt_initialize_for_cpu(0); kprintf("* Thread local1: %i\n", __tl1__); - kprintf("* Elf start: 0x%lx\n", &_ELF_START_); + kprintf("* Elf start: %p\n", &_ELF_START_); auto* ehdr = (Elf64_Ehdr*)&_ELF_START_; auto* phdr = (Elf64_Phdr*)((char*)ehdr + ehdr->e_phoff); Expects(phdr); @@ -157,8 +157,8 @@ void kernel_start(uintptr_t magic, uintptr_t addr) size_t size = &_ELF_END_ - &_ELF_START_; Expects(phdr[0].p_type == PT_LOAD); - printf("* Elf ident: %s, program headers:\n", ehdr->e_ident, ehdr); - kprintf("\tElf size: %i \n", size); + printf("* Elf ident: %s, program headers: %p\n", ehdr->e_ident, ehdr); + kprintf("\tElf size: %zu \n", size); for (int i = 0; i < ehdr->e_phnum; i++) { kprintf("\tPhdr %i @ %p, va_addr: 0x%lx \n", i, &phdr[i], phdr[i].p_vaddr); @@ -199,7 +199,7 @@ void kernel_start(uintptr_t magic, uintptr_t addr) int argc = 1; // Env vars - argv[2] = "USER=root"; + argv[2] = strdup("USER=root"); argv[3] = 0x0; memcpy(&argv[4], aux, sizeof(auxv_t) * 38); From 10d2731adc36878ce63fb4aafad96e815a9be9c1 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 13 Mar 2018 15:05:24 +0100 Subject: [PATCH 076/723] bundle: update to bundle with chainloader --- cmake/cross_compiled_libraries.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cross_compiled_libraries.cmake b/cmake/cross_compiled_libraries.cmake index 95d7020465..98186a0433 100644 --- a/cmake/cross_compiled_libraries.cmake +++ b/cmake/cross_compiled_libraries.cmake @@ -19,7 +19,7 @@ else(BUNDLE_LOC) ExternalProject_Add(PrecompiledLibraries PREFIX precompiled URL https://github.com/hioa-cs/IncludeOS/releases/download/v0.12.0-rc.2/IncludeOS_dependencies_v0-12-0_musl_libunwind_threaded.tar.gz - URL_HASH SHA1=9f3f8da09c1f15bdc2a7d3feea665a31cb9fe889 + URL_HASH SHA1=5a9daff15ed7b0fcd2a96f3c07ebef5aabf93371 CONFIGURE_COMMAND "" BUILD_COMMAND "" UPDATE_COMMAND "" From 506a69917e4317251c3ebad75daa8a739d42a99d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 13 Mar 2018 15:30:51 +0100 Subject: [PATCH 077/723] x86: removed ctors and dtors from sneaking into binary --- src/arch/x86_64/linker.ld | 45 +-------------------------------------- 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/src/arch/x86_64/linker.ld b/src/arch/x86_64/linker.ld index 75d1c94da5..65d08904b4 100644 --- a/src/arch/x86_64/linker.ld +++ b/src/arch/x86_64/linker.ld @@ -43,59 +43,16 @@ SECTIONS } PROVIDE( _TEXT_END_ = . ); - .init ALIGN(0x1000) : { - _INIT_START_ = .; - *(.init) - _INIT_END_ = .; - } - - .fini ALIGN(0x10) : { - *(.fini) - } - /* Global offset-table. For dynamic linking */ .got ALIGN(0x10) : { *(.got*) } /** - * .ctors, .dtors, .preinit_array, .init_array, .fini_array + * .preinit_array, .init_array, .fini_array * from GNU LD default linker script */ -.ctors : - { - _GCONSTR_START_ = .; - /* gcc uses crtbegin.o to find the start of - the constructors, so we make sure it is - first. Because this is a wildcard, it - doesn't matter if the user does not - actually link against crtbegin.o; the - linker won't look for a file to match a - wildcard. The wildcard also means that it - doesn't matter which directory crtbegin.o - is in. */ - KEEP (*crtbegin.o(.ctors)) - KEEP (*crtbegin?.o(.ctors)) - /* We don't want to include the .ctor section from - the crtend.o file until after the sorted ctors. - The .ctor section from the crtend file contains the - end of ctors marker and it must be last */ - KEEP (*(EXCLUDE_FILE (*crtend.o *crtend?.o ) .ctors)) - KEEP (*(SORT(.ctors.*))) - KEEP (*(.ctors)) - _GCONSTR_END_ = .; - } - - .dtors : - { - KEEP (*crtbegin.o(.dtors)) - KEEP (*crtbegin?.o(.dtors)) - KEEP (*(EXCLUDE_FILE (*crtend.o *crtend?.o ) .dtors)) - KEEP (*(SORT(.dtors.*))) - KEEP (*(.dtors)) - } - .preinit_array : { PROVIDE_HIDDEN (__preinit_array_start = .); From facdac5f3a48d7907e4c84f6528a1b15de969259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 14 Mar 2018 11:35:25 +0100 Subject: [PATCH 078/723] musl: Implemented uname and updated test --- src/musl/CMakeLists.txt | 2 +- src/musl/uname.cpp | 25 +++++++++++++++++++++ test/posix/integration/utsname/service.cpp | 26 ++++++++++++++-------- test/posix/integration/utsname/test.py | 21 +---------------- 4 files changed, 44 insertions(+), 30 deletions(-) create mode 100644 src/musl/uname.cpp diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index 094d6a01da..873fc013e2 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -20,7 +20,7 @@ set(MUSL_OBJECTS openat.cpp umask.cpp utimensat.cpp - + uname.cpp ) add_library(musl_syscalls STATIC ${MUSL_OBJECTS}) diff --git a/src/musl/uname.cpp b/src/musl/uname.cpp new file mode 100644 index 0000000000..73df0e6985 --- /dev/null +++ b/src/musl/uname.cpp @@ -0,0 +1,25 @@ +#include "common.hpp" +#include +#include + +static long sys_uname(struct utsname *buf) { + if(UNLIKELY(buf == nullptr)) + return -EFAULT; + + strcpy(buf->sysname, "IncludeOS"); + + strcpy(buf->nodename, "IncludeOS-node"); + + strcpy(buf->release, OS::version().c_str()); + + strcpy(buf->version, OS::version().c_str()); + + strcpy(buf->machine, ARCH); + + return 0; +} + +extern "C" +long syscall_SYS_uname(struct utsname *buf) { + return strace(sys_uname, "uname", buf); +} diff --git a/test/posix/integration/utsname/service.cpp b/test/posix/integration/utsname/service.cpp index c3a5b50ccb..75a1ca807c 100644 --- a/test/posix/integration/utsname/service.cpp +++ b/test/posix/integration/utsname/service.cpp @@ -1,6 +1,6 @@ // This file is a part of the IncludeOS unikernel - www.includeos.org // -// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences +// Copyright 2015-2018 Oslo and Akershus University College of Applied Sciences // and Alfred Bratterud // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,23 +15,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include int main() { struct utsname struct_test; - uname(&struct_test); - printf("Sysname: %s\n", struct_test.sysname); - printf("Nodename: %s\n", struct_test.nodename); - printf("Release: %s\n", struct_test.release); - printf("Version: %s\n", struct_test.version); - printf("Machine: %s\n", struct_test.machine); - printf("Something special to close with\n"); + CHECKSERT(uname(&struct_test) == 0, "uname with buffer returns no error"); + CHECKSERT(strcmp(struct_test.sysname, "IncludeOS") == 0, + "sysname is IncludeOS"); + CHECKSERT(strcmp(struct_test.nodename, "IncludeOS-node") == 0, + "nodename is IncludeOS-node"); + CHECKSERT(strcmp(struct_test.release, OS::version().c_str()) == 0, + "release is %s", OS::version().c_str()); + CHECKSERT(strcmp(struct_test.version, OS::version().c_str()) == 0, + "version is %s", OS::version().c_str()); + CHECKSERT(strcmp(struct_test.machine, ARCH) == 0, + "machine is %s", ARCH); + + CHECKSERT(uname(nullptr) == -1, "uname with nullptr returns error"); + CHECKSERT(errno == EFAULT, "error is EFAULT"); return 0; } void Service::start(const std::string&) { main(); + printf("SUCCESS\n"); } diff --git a/test/posix/integration/utsname/test.py b/test/posix/integration/utsname/test.py index 80de18e077..a9f8d1c18d 100755 --- a/test/posix/integration/utsname/test.py +++ b/test/posix/integration/utsname/test.py @@ -5,30 +5,11 @@ includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) -print 'includeos_src: {0}'.format(includeos_src) sys.path.insert(0,includeos_src) from vmrunner import vmrunner vm = vmrunner.vms[0] vm.cmake() -num_outputs = 0 - -def increment(line): - global num_outputs - num_outputs += 1 - print "num_outputs after increment: ", num_outputs - -def check_num_outputs(line): - assert(num_outputs == 5) - vmrunner.vms[0].exit(0, "All tests passed", True) - -vm.on_output("Sysname: IncludeOS", increment) -vm.on_output("Nodename: IncludeOS-node", increment) -vm.on_output("Release: v", increment) -vm.on_output("Version: v", increment) -vm.on_output("Machine: x86_64", increment) -vm.on_output("Something special to close with", check_num_outputs) - # Boot the VM, taking a timeout as parameter -vm.boot(20).clean() +vm.boot(10).clean() From 71870ee772a77b308162c83ca5bb0712f45ad6ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 15 Mar 2018 10:56:11 +0100 Subject: [PATCH 079/723] musl: Work on files --- src/CMakeLists.txt | 2 +- src/musl/open.cpp | 28 +++++++++++++++++++++++++++- src/posix/file_fd.cpp | 5 ++--- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index c5c4595455..75d4a79363 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -60,7 +60,7 @@ set(OS_OBJECTS fs/disk.cpp fs/filesystem.cpp fs/dirent.cpp fs/mbr.cpp fs/path.cpp fs/fat.cpp fs/fat_async.cpp fs/fat_sync.cpp fs/memdisk.cpp # POSIX - posix/fd.cpp + posix/fd.cpp posix/file_fd.cpp ) add_library(os STATIC ${OS_OBJECTS}) diff --git a/src/musl/open.cpp b/src/musl/open.cpp index 0b1b8254ef..9570128c47 100644 --- a/src/musl/open.cpp +++ b/src/musl/open.cpp @@ -2,14 +2,40 @@ #include #include +#include +#include static long sys_open(const char *pathname, int flags, mode_t mode = 0) { + if (UNLIKELY(pathname == nullptr)) + return -EFAULT; + + if (UNLIKELY(pathname[0] == 0)) + return -ENOENT; + try { auto& entry = fs::VFS::get(pathname); auto& fd = entry.open_fd(); return fd.get_id(); } - catch(const fs::VFS_err&) { + // Not FD_compatible, try dirent + catch(const fs::Err_bad_cast&) + { + try { + auto ent = fs::VFS::stat_sync(pathname); + if (ent.is_valid()) + { + auto& fd = FD_map::_open(ent); + return fd.get_id(); + } + return -ENOENT; + } + catch(...) + { + return -ENOENT; + } + } + catch(const fs::VFS_err&) + { // VFS error for one of many reasons (not mounted, not fd compatible etc) } catch(const std::bad_function_call&) { diff --git a/src/posix/file_fd.cpp b/src/posix/file_fd.cpp index 259187a0a0..342cb6e6a4 100644 --- a/src/posix/file_fd.cpp +++ b/src/posix/file_fd.cpp @@ -17,7 +17,6 @@ #include #include -#include int File_FD::read(void* p, size_t n) { auto buf = ent_.read(offset_, n); @@ -37,7 +36,7 @@ int File_FD::close() { int File_FD::lseek(off_t offset, int whence) { if ((whence != SEEK_SET) && (whence != SEEK_CUR) && (whence != SEEK_END)) { - PRINT("lseek(%lu, %d) == %d\n", offset, whence, -1); + //PRINT("lseek(%lu, %d) == %d\n", offset, whence, -1); errno = EINVAL; return -1; } @@ -54,6 +53,6 @@ int File_FD::lseek(off_t offset, int whence) } if (calculated_offset < 0) calculated_offset = 0; offset_ = calculated_offset; - PRINT("lseek(%lu, %d) == %d\n", offset, whence, offset_); + //PRINT("lseek(%lu, %d) == %d\n", offset, whence, offset_); return offset_; } From 39b775cd263fb5585e30b6a49d1344bb2774cb84 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 15 Mar 2018 14:56:07 +0100 Subject: [PATCH 080/723] bundle: build with -g and threaing off by default --- etc/build_llvm.sh | 4 ++-- etc/build_musl.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/build_llvm.sh b/etc/build_llvm.sh index 1862c4acfe..32e66c0757 100755 --- a/etc/build_llvm.sh +++ b/etc/build_llvm.sh @@ -5,7 +5,7 @@ ARCH=${ARCH:-x86_64} # CPU architecture. Alternatively x86_64 TARGET=$ARCH-elf # Configure target based on arch. Always ELF. BUILD_DIR=${BUILD_DIR:-~/IncludeOS_build/build_llvm} -INCLUDEOS_THREADING=${INCLUDEOS_THREADING:-ON} +INCLUDEOS_THREADING=${INCLUDEOS_THREADING:-OFF} musl_inc=$TEMP_INSTALL_DIR/$TARGET/include # path for newlib headers IncludeOS_posix=$INCLUDEOS_SRC/api/posix @@ -66,7 +66,7 @@ fi TRIPLE=$ARCH-pc-linux-elf -CXX_FLAGS="-std=c++14 -msse3 -mfpmath=sse -nostdlibinc -D_LIBCPP_HAS_MUSL_LIBC" +CXX_FLAGS="-std=c++14 -msse3 -g -mfpmath=sse -nostdlibinc -D_LIBCPP_HAS_MUSL_LIBC" # CMAKE configure step # diff --git a/etc/build_musl.sh b/etc/build_musl.sh index 5f2eefc5e4..c95f242fce 100755 --- a/etc/build_musl.sh +++ b/etc/build_musl.sh @@ -30,7 +30,7 @@ git apply $INCLUDEOS_SRC/etc/musl/endian.patch || true git checkout $musl_version make distclean || true -export CFLAGS="$CFLAGS -target $ARCH-pc-linux-elf" +export CFLAGS="$CFLAGS -g -target $ARCH-pc-linux-elf" ./configure --prefix=$TEMP_INSTALL_DIR/$TARGET --disable-shared --enable-debug --target=$TARGET #--enable-optimize=* make $num_jobs make install From 1aa6b95b3a8b2daeabeed219abdfd9e1e1c07ad2 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 15 Mar 2018 15:34:07 +0100 Subject: [PATCH 081/723] kernel: separate kernel- plugin and service global ctors --- src/arch/x86_64/linker.ld | 28 ++++++++++++++++++++- src/kernel/os.cpp | 37 ++++++++++++++++++++++++++-- src/platform/x86_pc/kernel_start.cpp | 29 +++++++++++++++------- 3 files changed, 82 insertions(+), 12 deletions(-) diff --git a/src/arch/x86_64/linker.ld b/src/arch/x86_64/linker.ld index 65d08904b4..dba947b00b 100644 --- a/src/arch/x86_64/linker.ld +++ b/src/arch/x86_64/linker.ld @@ -60,12 +60,38 @@ SECTIONS PROVIDE_HIDDEN (__preinit_array_end = .); } +/** + * Global constructors + * Constructors are split into groups allowing the OS to use global ctors + * before the OS itself is initialized, while delaying the calls to service constructors + * until as much of the OS / C++ runtime as possible is ready. + */ + + /* OS / stdlib constructors */ .init_array : { PROVIDE_HIDDEN (__init_array_start = .); + */x86_64/lib/lib*.a:*(.init_array*) + */x86_64/platform/lib*.a:*(.init_array*) + */x86_64/drivers/lib*.a:*(.init_array*) + PROVIDE_HIDDEN (__init_array_end = .); + } + + /* Plugin constructors */ + .plugin_ctors : + { + PROVIDE_HIDDEN (__plugin_ctors_start = .); + */x86_64/plugins/lib*.a:*(.init_array*) + PROVIDE_HIDDEN (__plugin_ctors_end = .); + } + + /* All other constructors */ + .service_ctors : + { + PROVIDE_HIDDEN (__service_ctors_start = .); KEEP (*(SORT_BY_INIT_PRIORITY(.init_array.*) SORT_BY_INIT_PRIORITY(.ctors.*))) KEEP (*(.init_array EXCLUDE_FILE (*crtbegin.o *crtbegin?.o *crtend.o *crtend?.o ) .ctors)) - PROVIDE_HIDDEN (__init_array_end = .); + PROVIDE_HIDDEN (__service_ctors_end = .); } .fini_array : diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 0b27458873..ce24d1101e 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -89,6 +89,20 @@ const char* OS::cmdline_args() noexcept { return cmdline; } +extern char __service_ctors_start; +extern char __service_ctors_end; +extern char __plugin_ctors_start; +extern char __plugin_ctors_end; + +static int __run_ctors(uintptr_t begin, uintptr_t end) +{ + int i = 0; + for (; begin < end; begin += sizeof(void(*)()), i++) { + (*(void (**)(void)) begin )(); + } + return i; +} + void OS::register_plugin(Plugin delg, const char* name){ MYINFO("Registering plugin %s", name); plugins.emplace_back(delg, name); @@ -128,15 +142,26 @@ void OS::post_start() // Custom initialization functions MYINFO("Initializing plugins"); - // the boot sequence is over when we get to plugins/Service::start - OS::boot_sequence_passed_ = true; + // Run plugin constructors + __run_ctors((uintptr_t)&__plugin_ctors_start, (uintptr_t)&__plugin_ctors_end); + + + // Run plugins PROFILE("Plugins init"); for (auto plugin : plugins) { INFO2("* Initializing %s", plugin.name); plugin.func(); } + MYINFO("Running service constructors"); + FILLINE('-'); + // the boot sequence is over when we get to plugins/Service::start + OS::boot_sequence_passed_ = true; + + // Run service constructors + __run_ctors((uintptr_t)&__service_ctors_start, (uintptr_t)&__service_ctors_end); + PROFILE("Service::start"); // begin service start FILLINE('='); @@ -157,8 +182,16 @@ __attribute__((weak)) bool os_enable_boot_logging = false; __attribute__((weak)) bool os_default_stdout = false; +extern bool __libc_initialized; + void OS::print(const char* str, const size_t len) { + + if (UNLIKELY(! __libc_initialized)) { + OS::default_stdout(str, len); + return; + } + for (auto& callback : os_print_handlers) { if (os_enable_boot_logging || OS::is_booted() || OS::is_panicking()) callback(str, len); diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 5b9c9d4f89..b426a42671 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -47,7 +47,13 @@ extern char _ELF_END_; extern char _INIT_START_; extern char _FINI_START_; -static thread_local int __tl1__ = 42; +thread_local int __tl1__ = 42; + +uintptr_t __grub_magic = 0xc001; +uintptr_t __grub_addr = 0x7001; + +static int __global_ctors_ok = 0; +bool __libc_initialized = false; void _init_bss() @@ -57,13 +63,17 @@ void _init_bss() __builtin_memset(&_BSS_START_, 0, &_BSS_END_ - &_BSS_START_); } -uintptr_t __grub_magic = 0xc001; -uintptr_t __grub_addr = 0x7001; -bool __libc_initialized = false; + +__attribute__((constructor)) +static void global_ctor_test(){ + __global_ctors_ok = 42; +} int kernel_main(int, char * *, char * *) { kprintf(" libc initialization complete \n"); + Expects(__global_ctors_ok == 42); __libc_initialized = true; + Expects(__tl1__ == 42); Elf_binary elf{{(char*)&_ELF_START_, &_ELF_END_ - &_ELF_START_}}; Expects(elf.is_ELF() && "ELF header intact"); @@ -138,14 +148,15 @@ void kernel_start(uintptr_t magic, uintptr_t addr) kprintf("* Init heap\n"); _init_heap(free_mem_begin); - /// init ELF / backtrace functionality - _init_elf_parser(); - - // init syscalls + kprintf("* Init syscalls\n"); _init_syscalls(); - // Initialize CPU exceptions + kprintf("* Init CPU exceptions\n"); x86::idt_initialize_for_cpu(0); + + kprintf("* Init ELF parser\n"); + _init_elf_parser(); + kprintf("* Thread local1: %i\n", __tl1__); kprintf("* Elf start: %p\n", &_ELF_START_); From a0aa4cefaaeb52c2ea501a75e328cbb0ee44c87f Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 15 Mar 2018 15:37:20 +0100 Subject: [PATCH 082/723] test: update paging test to work with new /dev/random --- test/kernel/integration/paging/CMakeLists.txt | 2 +- test/kernel/integration/paging/service.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/kernel/integration/paging/CMakeLists.txt b/test/kernel/integration/paging/CMakeLists.txt index 6cba91c174..f62c1f4660 100644 --- a/test/kernel/integration/paging/CMakeLists.txt +++ b/test/kernel/integration/paging/CMakeLists.txt @@ -21,7 +21,7 @@ set(DRIVERS boot_logger ) -set(PLUGINS ) +set(PLUGINS vfs) # STATIC LIBRARIES: set(LIBRARIES diff --git a/test/kernel/integration/paging/service.cpp b/test/kernel/integration/paging/service.cpp index 2c4b932a57..95b49c7ed2 100644 --- a/test/kernel/integration/paging/service.cpp +++ b/test/kernel/integration/paging/service.cpp @@ -401,6 +401,7 @@ void map_non_aligned(){ int main() { + void(*heap_code)() = (void(*)()) malloc(42); Expects(Byte_r{std::numeric_limits::max()}.to_string() == "2.000_GiB"); Expects(Byte_r{std::numeric_limits::max()}.to_string() == "16777216.000_TiB"); @@ -476,7 +477,6 @@ int main() ((void(*)())(&protected_page[magic->i]))(); } - void(*heap_code)() = (void(*)()) malloc(42); if (magic->reboots == 3){ // Verify XD Expects(magic->last_error = Page_fault); From 551d47bc21cd5e2e8b13bd584e29241731e4d193 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 15 Mar 2018 17:47:45 +0100 Subject: [PATCH 083/723] lib: Zero RAM during LiveUpdate --- lib/LiveUpdate/hotswap64.asm | 16 +++++++++++++++- lib/LiveUpdate/update.cpp | 5 +++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/LiveUpdate/hotswap64.asm b/lib/LiveUpdate/hotswap64.asm index 1f001a4f6c..8b85ef0a0a 100644 --- a/lib/LiveUpdate/hotswap64.asm +++ b/lib/LiveUpdate/hotswap64.asm @@ -33,7 +33,8 @@ ALIGN 16 ;; RSI: const char* base, ;; RDX: size_t len, ;; RCX: void* entry_function, -;; R8: void* reset_data) +;; R8: void* reset_data, +;; R9: void* liu_storage) hotswap_amd64: ;; save soft reset data location and entry function mov rax, r8 @@ -47,6 +48,19 @@ hotswap_amd64: cld rep movsb + ;; clear heap + kernel remains + ;; RDI already at end of new kernel + mov rax, 0 + mov rcx, r9 ;; LU location + sub rcx, rdi ;; - KERN_END + rep stosb + + ;; clear stack + mov rdi, 0x100000 + mov rax, 0 + mov rcx, 0x100000 + rep stosb + begin_enter_protected: ; load 64-bit GDTR with 32-bit entries lgdt [gdtr64] diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index 1f127addf4..6c9ce6d801 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -43,7 +43,7 @@ extern "C" void solo5_exec(const char*, size_t); static void* HOTSWAP_AREA = (void*) 0x8000; extern "C" void hotswap(const char*, int, char*, uintptr_t, void*); extern "C" char __hotswap_length; -extern "C" void hotswap64(char*, const char*, int, uintptr_t, void*); +extern "C" void hotswap64(char*, const char*, int, uintptr_t, void*, void*); extern uint32_t hotswap64_len; extern void __x86_init_paging(void*); extern "C" void* __os_store_soft_reset(const void*, size_t); @@ -241,7 +241,8 @@ void LiveUpdate::exec(const buffer_t& blob) // copy hotswapping function to sweet spot memcpy(HOTSWAP_AREA, (void*) &hotswap64, hotswap64_len); /// the end - ((decltype(&hotswap64)) HOTSWAP_AREA)(phys_base, bin_data, bin_len, start_offset, sr_data); + char* end_loc = storage_area - 0x110000; + ((decltype(&hotswap64)) HOTSWAP_AREA)(phys_base, bin_data, bin_len, start_offset, sr_data, end_loc); # else # error "Unimplemented architecture" # endif From 8d9776baa6b4da813b72c07c17865d5353c96363 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 16 Mar 2018 11:02:25 +0100 Subject: [PATCH 084/723] musl: Avoid nullptr in strace --- src/musl/common.hpp | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/musl/common.hpp b/src/musl/common.hpp index 7af2d70f63..416dbdf650 100644 --- a/src/musl/common.hpp +++ b/src/musl/common.hpp @@ -22,10 +22,26 @@ template inline constexpr auto& pr_param(std::ostream& out, L lhs, Args&&... rest){ if constexpr (sizeof...(rest) > 0) { - out << lhs << ", "; + // avoid writing nullptr to std out + if constexpr(std::is_pointer_v) { + if(lhs != nullptr) out << lhs << ", "; + else out << "nullptr, "; + } + else { + out << lhs << ", "; + } pr_param(out, rest...); - } else { - out << lhs; + } + else + { + // avoid writing nullptr to std out + if constexpr(std::is_pointer_v) { + if(lhs != nullptr) out << lhs; + else out << "nullptr"; + } + else { + out << lhs; + } } return out; } From 1d4d1ee86e3d68d72d6291a44045528c485c9a1d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 16 Mar 2018 18:58:04 +0100 Subject: [PATCH 085/723] drivers: Move virtio console to drivers and modernize --- src/drivers/CMakeLists.txt | 5 +- src/drivers/virtiocon.cpp | 163 ++++++++++++++++ .../console.hpp => src/drivers/virtiocon.hpp | 77 ++++---- src/virtio/console.cpp | 174 ------------------ 4 files changed, 202 insertions(+), 217 deletions(-) create mode 100644 src/drivers/virtiocon.cpp rename api/virtio/console.hpp => src/drivers/virtiocon.hpp (67%) delete mode 100644 src/virtio/console.cpp diff --git a/src/drivers/CMakeLists.txt b/src/drivers/CMakeLists.txt index 5032d4a1bb..8352e7c846 100644 --- a/src/drivers/CMakeLists.txt +++ b/src/drivers/CMakeLists.txt @@ -22,6 +22,9 @@ add_library(ide_writeonly STATIC ide.cpp) set_target_properties(ide_writeonly PROPERTIES COMPILE_FLAGS "-DIDE_ENABLE_WRITE") add_dependencies(ide_writeonly PrecompiledLibraries) +add_library(virtiocon STATIC virtiocon.cpp) +add_dependencies(virtiocon PrecompiledLibraries) + add_library(virtioblk STATIC virtioblk.cpp) add_dependencies(virtioblk PrecompiledLibraries) @@ -60,7 +63,7 @@ set(CMAKE_INSTALL_MESSAGE LAZY) # to avoid spam install(TARGETS default_stdout ide_readwrite ide_readonly ide_writeonly - virtionet virtioblk + virtionet virtioblk virtiocon vmxnet3 e1000 ip4_reassembly heap_debugging diff --git a/src/drivers/virtiocon.cpp b/src/drivers/virtiocon.cpp new file mode 100644 index 0000000000..ef37175f36 --- /dev/null +++ b/src/drivers/virtiocon.cpp @@ -0,0 +1,163 @@ +#include "virtiocon.hpp" +#include +#include +#include +#include + +#ifdef VCON_DEBUG +#define VCPRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define VCPRINT(fmt, ...) /* fmt */ +#endif + +#define VIRTIO_CONSOLE_F_SIZE 0 +#define VIRTIO_CONSOLE_F_MULTIPORT 1 +#define VIRTIO_CONSOLE_F_EMERG_WRITE 2 + +#define VIRTIO_CONSOLE_DEVICE_READY 0 +#define VIRTIO_CONSOLE_DEVICE_ADD 1 +#define VIRTIO_CONSOLE_DEVICE_REMOVE 2 +#define VIRTIO_CONSOLE_PORT_READY 3 +#define VIRTIO_CONSOLE_CONSOLE_PORT 4 +#define VIRTIO_CONSOLE_RESIZE 5 +#define VIRTIO_CONSOLE_PORT_OPEN 6 +#define VIRTIO_CONSOLE_PORT_NAME 7 +#define FEAT(x) (1 << x) + +static int index_counter = 0; +VirtioCon::VirtioCon(hw::PCI_Device& d) +: Virtio(d), m_index(index_counter) +{ + index_counter++; + INFO("VirtioCon", "Driver initializing"); + + new (&rx) Virtio::Queue(device_name() + ".rx_q", queue_size(0), 0, iobase()); + new (&tx) Virtio::Queue(device_name() + ".tx_q", queue_size(1), 1, iobase()); + new (&ctl_rx) Virtio::Queue(device_name() + ".ctl_rx_q", queue_size(2), 2, iobase()); + new (&ctl_tx) Virtio::Queue(device_name() + ".ctl_tx_q", queue_size(3), 3, iobase()); + + const uint32_t needed_features = + FEAT(VIRTIO_CONSOLE_F_MULTIPORT); + negotiate_features(needed_features); + + CHECK(features() & FEAT(VIRTIO_CONSOLE_F_SIZE), + "Valid console dimensions"); + CHECK(features() & FEAT(VIRTIO_CONSOLE_F_MULTIPORT), + "Multiple ports support"); + CHECK(features() & FEAT(VIRTIO_CONSOLE_F_EMERG_WRITE), + "Emergency write support"); + + CHECK ((features() & needed_features) == needed_features, + "Negotiated needed features"); + + // Step 1 - Initialize queues + auto success = assign_queue(0, rx.queue_desc()); + CHECK(success, "Receive queue assigned (%p) to device", + rx.queue_desc()); + + success = assign_queue(1, tx.queue_desc()); + CHECK(success, "Transmit queue assigned (%p) to device", + tx.queue_desc()); + + success = assign_queue(2, ctl_rx.queue_desc()); + CHECK(success, "Control rx queue assigned (%p) to device", + ctl_rx.queue_desc()); + + success = assign_queue(3, ctl_tx.queue_desc()); + CHECK(success, "Control tx queue assigned (%p) to device", + ctl_tx.queue_desc()); + + // Step 3 - Fill receive queue with buffers + INFO("VirtioCon", "Queue size rx: %d tx: %d\n", + rx.size(), tx.size()); + + // Get device configuration + get_config(); + + // Signal setup complete. + setup_complete((features() & needed_features) == needed_features); + CHECK((features() & needed_features) == needed_features, "Signalled driver OK"); + + if (has_msix()) + { + assert(get_msix_vectors() >= 3); + auto& irqs = this->get_irqs(); + Events::get().subscribe(irqs[0], {this, &VirtioCon::msix_recv_handler}); + Events::get().subscribe(irqs[1], {this, &VirtioCon::msix_xmit_handler}); + Events::get().subscribe(irqs[2], {this, &VirtioCon::event_handler}); + } + else + { + auto irq = Virtio::get_legacy_irq(); + Events::get().subscribe(irq, {this, &VirtioCon::event_handler}); + } + + // Done + INFO("VirtioCon", "Console with size (%u, %u), %u ports", + config.cols, config.rows, config.max_nr_ports); + rx.kick(); +} + +void VirtioCon::get_config() +{ + Virtio::get_config(&config, sizeof(console_config)); +} + +void VirtioCon::event_handler() +{ + int isr = hw::inp(iobase() + VIRTIO_PCI_ISR); + VCPRINT(" ISR: %#x\n", isr); + + if (isr & 1) + { + msix_recv_handler(); + msix_xmit_handler(); + } + if (isr & 2) + { + get_config(); + VCPRINT("\t Configuration change: %#x\n", config.status); + } +} + +void VirtioCon::msix_recv_handler() +{ + rx.disable_interrupts(); + + while (rx.new_incoming()) + { + auto res = rx.dequeue(); + if (res.data() != nullptr) + { + printf("virtiocon rx: %.*s\n", (int) res.size(), res.data()); + } + else + { + // acknowledgement + printf("No data, len = %lu\n", res.size()); + } + } + rx.enable_interrupts(); +} +void VirtioCon::msix_xmit_handler() +{ + tx.disable_interrupts(); + + while (tx.new_incoming()) + { + auto res = tx.dequeue(); + free(res.data()); + } + tx.enable_interrupts(); +} + +void VirtioCon::write(const void* data, size_t len) +{ + uint8_t* heapdata = new uint8_t[len]; + memcpy(heapdata, data, len); + + const Token token {{ heapdata, len }, Token::OUT }; + std::array tokens { token }; + tx.enqueue(tokens); + tx.kick(); +} diff --git a/api/virtio/console.hpp b/src/drivers/virtiocon.hpp similarity index 67% rename from api/virtio/console.hpp rename to src/drivers/virtiocon.hpp index 951789bd9b..7ce845abc5 100644 --- a/api/virtio/console.hpp +++ b/src/drivers/virtiocon.hpp @@ -6,9 +6,9 @@ // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,32 +20,36 @@ #define VIRTIO_CONSOLE_HPP #include -#include "../hw/pci_device.hpp" -#include "virtio.hpp" #include +#include +#include /** * http://docs.oasis-open.org/virtio/virtio/v1.0/csprd05/virtio-v1.0-csprd05.html#x1-2180003 - * - * The virtio console device is a simple device for data input - * and output. A device MAY have one or more ports. Each port has + * + * The virtio console device is a simple device for data input + * and output. A device MAY have one or more ports. Each port has * a pair of input and output virtqueues. Moreover, a device has a - * pair of control IO virtqueues. The control virtqueues are used - * to communicate information between the device and the driver - * about ports being opened and closed on either side of the - * connection, indication from the device about whether a - * particular port is a console port, adding new ports, port - * hot-plug/unplug, etc., and indication from the driver about - * whether a port or a device was successfully added, - * port open/close, etc. For data IO, one or more empty buffers are - * placed in the receive queue for incoming data and outgoing - * characters are placed in the transmit queue. - * + * pair of control IO virtqueues. The control virtqueues are used + * to communicate information between the device and the driver + * about ports being opened and closed on either side of the + * connection, indication from the device about whether a + * particular port is a console port, adding new ports, port + * hot-plug/unplug, etc., and indication from the driver about + * whether a port or a device was successfully added, + * port open/close, etc. For data IO, one or more empty buffers are + * placed in the receive queue for incoming data and outgoing + * characters are placed in the transmit queue. + * **/ class VirtioCon : public Virtio { public: + std::string device_name() const { + return "virtiocon" + std::to_string(m_index); + } + // returns the sizes of this console size_t rows() const noexcept { @@ -55,12 +59,12 @@ class VirtioCon : public Virtio { return config.cols; } - + void write (const void* data, size_t len); - + /** Constructor. @param pcidev an initialized PCI device. */ VirtioCon(hw::PCI_Device& pcidev); - + private: struct console_config { @@ -69,46 +73,35 @@ class VirtioCon : public Virtio uint32_t max_nr_ports; uint32_t emerg_wr; }; - + struct console_control { uint32_t id; // port number, but lets call it id so we need a comment to explain what it is :) uint16_t event; uint16_t value; }; - + struct console_resize { uint16_t cols; uint16_t rows; }; - + /** Get virtio PCI config. @see Virtio::get_config.*/ void get_config(); - - /** Service the RX Queue. - Push incoming data up to linklayer, dequeue RX buffers. */ - void service_RX(); - - /** Service the TX Queue - Dequeue used TX buffers. @note: This function does not take any - responsibility for memory management. */ - void service_TX(); - - /** - * Handle device IRQ. - * Will look for config. changes and service RX/TX queues as necessary. - **/ - void irq_handler(); - + + void event_handler(); + void msix_recv_handler(); + void msix_xmit_handler(); + Virtio::Queue rx; // 0 Virtio::Queue tx; // 1 Virtio::Queue ctl_rx; // 2 Virtio::Queue ctl_tx; // 3 - + // configuration as read from paravirtual PCI device console_config config; - + const int m_index = 0; }; #endif diff --git a/src/virtio/console.cpp b/src/virtio/console.cpp deleted file mode 100644 index 2433ffe700..0000000000 --- a/src/virtio/console.cpp +++ /dev/null @@ -1,174 +0,0 @@ -#define DEBUG -#define DEBUG2 -#include - -#include -#include -#include -#include -extern "C" -{ void panic(const char*); } - -#define VIRTIO_CONSOLE_F_SIZE 0 -#define VIRTIO_CONSOLE_F_MULTIPORT 1 -#define VIRTIO_CONSOLE_F_EMERG_WRITE 2 - -#define VIRTIO_CONSOLE_DEVICE_READY 0 -#define VIRTIO_CONSOLE_DEVICE_ADD 1 -#define VIRTIO_CONSOLE_DEVICE_REMOVE 2 -#define VIRTIO_CONSOLE_PORT_READY 3 -#define VIRTIO_CONSOLE_CONSOLE_PORT 4 -#define VIRTIO_CONSOLE_RESIZE 5 -#define VIRTIO_CONSOLE_PORT_OPEN 6 -#define VIRTIO_CONSOLE_PORT_NAME 7 - - -#define FEAT(x) (1 << x) - -VirtioCon::VirtioCon(hw::PCI_Device& d) -: Virtio(d), - rx(queue_size(0), 0, iobase()), - tx(queue_size(1), 1, iobase()), - ctl_rx(queue_size(2), 2, iobase()), - ctl_tx(queue_size(3), 3, iobase()) -{ - INFO("VirtioCon", "Driver initializing"); - - uint32_t needed_features = - FEAT(VIRTIO_CONSOLE_F_MULTIPORT); - negotiate_features(needed_features); - - CHECK(features() & FEAT(VIRTIO_CONSOLE_F_SIZE), - "Valid console dimensions"); - CHECK(features() & FEAT(VIRTIO_CONSOLE_F_MULTIPORT), - "Multiple ports support"); - CHECK(features() & FEAT(VIRTIO_CONSOLE_F_EMERG_WRITE), - "Emergency write support"); - - CHECK ((features() & needed_features) == needed_features, - "Negotiated needed features"); - - // Step 1 - Initialize queues - auto success = assign_queue(0, (uint32_t) rx.queue_desc()); - CHECK(success, "Receive queue assigned (0x%x) to device", - (uint32_t) rx.queue_desc()); - - success = assign_queue(1, (uint32_t) tx.queue_desc()); - CHECK(success, "Transmit queue assigned (0x%x) to device", - (uint32_t) tx.queue_desc()); - - success = assign_queue(2, (uint32_t) ctl_rx.queue_desc()); - CHECK(success, "Control rx queue assigned (0x%x) to device", - (uint32_t) ctl_rx.queue_desc()); - - success = assign_queue(3, (uint32_t) ctl_tx.queue_desc()); - CHECK(success, "Control tx queue assigned (0x%x) to device", - (uint32_t) ctl_tx.queue_desc()); - - /* - success = assign_queue(4, (uint32_t) rx1.queue_desc()); - CHECK(success, "rx1 queue assigned (0x%x) to device", - (uint32_t) rx1.queue_desc()); - - success = assign_queue(5, (uint32_t) tx1.queue_desc()); - CHECK(success, "tx1 queue assigned (0x%x) to device", - (uint32_t) tx1.queue_desc()); - */ - - // Step 3 - Fill receive queue with buffers - INFO("VirtioCon", "Queue size rx: %d tx: %d\n", - rx.size(), tx.size()); - - // Get device configuration - get_config(); - - // Signal setup complete. - setup_complete((features() & needed_features) == needed_features); - CHECK((features() & needed_features) == needed_features, "Signalled driver OK"); - - // Hook up IRQ handler (inherited from Virtio) - auto del(delegate{this, &VirtioCon::irq_handler}); - IRQ_manager::subscribe(irq(), del); - IRQ_manager::enable_irq(irq()); - - // Done - INFO("VirtioCon", "Console with size (%u, %u), %u ports", - config.cols, config.rows, config.max_nr_ports); - rx.kick(); -} - -void VirtioCon::get_config() -{ - Virtio::get_config(&config, sizeof(console_config)); -} - -void VirtioCon::irq_handler() -{ - debug2(" IRQ handler\n"); - - //Virtio Std. § 4.1.5.5, steps 1-3 - - // Step 1. read ISR - unsigned char isr = hw::inp(iobase() + VIRTIO_PCI_ISR); - - // Step 2. A) - one of the queues have changed - if (isr & 1) - { - // This now means service RX & TX interchangeably - service_RX(); - } - - // Step 2. B) - if (isr & 2) - { - debug("\t Configuration change:\n"); - - //debug("\t Old status: 0x%x\n", config.status); - get_config(); - //debug("\t New status: 0x%x \n", config.status); - } - IRQ_manager::eoi(irq()); -} - -void VirtioCon::service_RX() -{ - rx.disable_interrupts(); - - while (rx.new_incoming()) - { - uint32_t len = 0; - char* condata = (char*) rx.dequeue(&len); - - uint32_t dontcare; - rx.dequeue(&dontcare); - - if (condata) - { - //printf("service_RX() received %u bytes from virtio console\n", len); - //printf("Data: %s\n", condata); - //vbr->handler(0, vbr->sector); - } - else - { - // acknowledgement - //printf("No data, just len = %d\n", len); - } - } - - rx.enable_interrupts(); -} - -void VirtioCon::write ( - const void* data, - size_t len) -{ - char* heapdata = new char[len]; - memcpy(heapdata, data, len); - - scatterlist sg[1]; - sg[0].data = (void*) heapdata; - sg[0].size = len; // +1? - - tx.enqueue(sg, 1, 0, (void*) data); - tx.kick(); -} From 80a01ef33bdad8a477f299ef594efd7a0a59b99f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Sun, 18 Mar 2018 19:49:00 +0100 Subject: [PATCH 086/723] test: Minor changes to LiveUpdate test --- lib/LiveUpdate/hotswap64.asm | 2 +- test/kernel/integration/LiveUpdate/CMakeLists.txt | 1 + test/kernel/integration/LiveUpdate/liu.hpp | 2 +- test/kernel/integration/LiveUpdate/server.hpp | 2 +- test/kernel/integration/LiveUpdate/service.cpp | 2 ++ test/kernel/integration/LiveUpdate/vm.json | 2 +- 6 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/LiveUpdate/hotswap64.asm b/lib/LiveUpdate/hotswap64.asm index 8b85ef0a0a..5c053d0332 100644 --- a/lib/LiveUpdate/hotswap64.asm +++ b/lib/LiveUpdate/hotswap64.asm @@ -52,7 +52,7 @@ hotswap_amd64: ;; RDI already at end of new kernel mov rax, 0 mov rcx, r9 ;; LU location - sub rcx, rdi ;; - KERN_END + sub rcx, rdi ;; - end of new kernel rep stosb ;; clear stack diff --git a/test/kernel/integration/LiveUpdate/CMakeLists.txt b/test/kernel/integration/LiveUpdate/CMakeLists.txt index 27b9800569..477ee3e0e7 100644 --- a/test/kernel/integration/LiveUpdate/CMakeLists.txt +++ b/test/kernel/integration/LiveUpdate/CMakeLists.txt @@ -24,6 +24,7 @@ set(SOURCES # DRIVERS / PLUGINS: set(DRIVERS virtionet + #boot_logger ) set(PLUGINS diff --git a/test/kernel/integration/LiveUpdate/liu.hpp b/test/kernel/integration/LiveUpdate/liu.hpp index 0d8b18a3fb..689bf82aa6 100644 --- a/test/kernel/integration/LiveUpdate/liu.hpp +++ b/test/kernel/integration/LiveUpdate/liu.hpp @@ -37,7 +37,7 @@ void setup_liveupdate_server(net::Inet& inet, // run live update process liu::LiveUpdate::exec(buffer, "test", save_function); } - catch (std::exception& err) + catch (const std::exception& err) { liu::LiveUpdate::restore_environment(); printf("Live update failed:\n%s\n", err.what()); diff --git a/test/kernel/integration/LiveUpdate/server.hpp b/test/kernel/integration/LiveUpdate/server.hpp index 4a440f34b3..1d2c36a6d5 100644 --- a/test/kernel/integration/LiveUpdate/server.hpp +++ b/test/kernel/integration/LiveUpdate/server.hpp @@ -17,7 +17,7 @@ void server(net::Inet& inet, [callback, port] (auto conn) { auto* buffer = new liu::buffer_t; - buffer->reserve(3*1024*1024); + buffer->reserve(4*1024*1024); printf("Receiving blob on port %u\n", port); // retrieve binary diff --git a/test/kernel/integration/LiveUpdate/service.cpp b/test/kernel/integration/LiveUpdate/service.cpp index 03548c2943..db4189f020 100644 --- a/test/kernel/integration/LiveUpdate/service.cpp +++ b/test/kernel/integration/LiveUpdate/service.cpp @@ -22,9 +22,11 @@ using storage_func_t = liu::LiveUpdate::storage_func; extern storage_func_t begin_test_boot(); +extern bool LIVEUPDATE_PERFORM_SANITY_CHECKS; void Service::start() { + //LIVEUPDATE_PERFORM_SANITY_CHECKS = false; OS::set_panic_action(OS::Panic_action::halt); auto func = begin_test_boot(); diff --git a/test/kernel/integration/LiveUpdate/vm.json b/test/kernel/integration/LiveUpdate/vm.json index 3e3dd609fa..9b1533e099 100644 --- a/test/kernel/integration/LiveUpdate/vm.json +++ b/test/kernel/integration/LiveUpdate/vm.json @@ -3,5 +3,5 @@ "net" : [ {"device" : "virtio"} ], - "mem" : 48 + "mem" : 64 } From cf04d3a96c93cc9edbf16d97f9684d608aa083f7 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Sun, 18 Mar 2018 20:51:02 +0100 Subject: [PATCH 087/723] Fix warnings in multiple sources files --- CMakeLists.txt | 2 +- api/kernel/terminal.hpp | 10 +++-- api/util/alloc_lstack.hpp | 6 +-- api/util/statman.hpp | 2 + lib/microLB/micro_lb/autoconf.cpp | 2 + src/crt/cxx_abi.cpp | 2 +- src/drivers/ide.cpp | 2 +- src/drivers/solo5blk.cpp | 2 +- src/drivers/solo5blk.hpp | 3 -- src/drivers/virtioblk.cpp | 6 +-- src/drivers/virtionet.cpp | 2 +- src/kernel/context.cpp | 8 +++- src/kernel/multiboot.cpp | 68 +++++++++++++++++-------------- src/kernel/profile.cpp | 6 +-- src/kernel/syscalls.cpp | 1 - src/musl/chmod.cpp | 5 +-- src/musl/creat.cpp | 2 +- src/musl/exit.cpp | 7 +++- src/musl/fchmod.cpp | 3 +- src/musl/fchmodat.cpp | 4 +- src/musl/fcntl.cpp | 2 +- src/musl/futex.cpp | 4 +- src/musl/getdents.cpp | 6 +-- src/musl/getrlimit.cpp | 5 +-- src/musl/ioctl.cpp | 3 +- src/musl/lseek.cpp | 8 ++-- src/musl/mkdir.cpp | 2 +- src/musl/mkdirat.cpp | 2 +- src/musl/mknod.cpp | 3 +- src/musl/mknodat.cpp | 3 +- src/musl/mmap.cpp | 20 ++++----- src/musl/open.cpp | 4 +- src/musl/openat.cpp | 6 +-- src/musl/poll.cpp | 6 +-- src/musl/readv.cpp | 5 +-- src/musl/set_tid_address.cpp | 2 - src/musl/sigmask.cpp | 39 +++++++++--------- src/musl/syscall_n.cpp | 7 ++-- src/musl/tkill.cpp | 7 ++-- src/musl/utimensat.cpp | 18 ++++---- src/net/buffer_store.cpp | 3 -- src/net/ip4/icmp4.cpp | 3 +- src/net/ip4/ip4.cpp | 2 +- src/net/openssl/init.cpp | 1 - src/platform/x86_pc/smp.cpp | 2 + src/plugins/unik.cpp | 2 +- src/posix/fd.cpp | 6 +-- src/util/async.cpp | 2 +- 48 files changed, 160 insertions(+), 156 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index cda9342253..2e11674674 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,7 +63,7 @@ set(CAPABS "${CAPABS} -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_P # * NO_DEBUG disables output from the debug macro # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE") -set(WARNS "-Wall -Wextra") +set(WARNS "-Wall -Wextra") # -Werror # configure options option(debug "Build with debugging symbols (OBS: Dramatically increases binary size)" OFF) diff --git a/api/kernel/terminal.hpp b/api/kernel/terminal.hpp index c72cd35281..4c37792b97 100644 --- a/api/kernel/terminal.hpp +++ b/api/kernel/terminal.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -41,11 +42,14 @@ class Terminal { Terminal(Connection_ptr); - template - void write(const char* str, Args&&... args) + __attribute__((format (printf, 2, 3))) + void write(const char* str, ...) { + va_list args; + va_start(args, str); char buffer[1024]; - int bytes = snprintf(buffer, 1024, str, args...); + int bytes = vsnprintf(buffer, 1024, str, args); + va_end(args); stream->write(buffer, bytes); } diff --git a/api/util/alloc_lstack.hpp b/api/util/alloc_lstack.hpp index 9df14ca1e9..3a5db0cd14 100644 --- a/api/util/alloc_lstack.hpp +++ b/api/util/alloc_lstack.hpp @@ -49,9 +49,9 @@ class Lstack { public: struct Chunk { - Chunk* next; - size_t size; - Chunk(Chunk* n, size_t s) + Chunk* next; + ssize_t size; + Chunk(Chunk* n, ssize_t s) : next(n), size(s) { Expects(s >= Min); diff --git a/api/util/statman.hpp b/api/util/statman.hpp index 1e5cc71fe5..fb4df08aef 100644 --- a/api/util/statman.hpp +++ b/api/util/statman.hpp @@ -184,7 +184,9 @@ class Statman { Stat* end_stats_; MemBitmap::word* bdata = nullptr; MemBitmap bitmap; +#ifndef INCLUDEOS_SINGLE_THREADED spinlock_t stlock = 0; +#endif Statman(const Statman& other) = delete; Statman(const Statman&& other) = delete; diff --git a/lib/microLB/micro_lb/autoconf.cpp b/lib/microLB/micro_lb/autoconf.cpp index 751af9a87b..c0016bfb41 100644 --- a/lib/microLB/micro_lb/autoconf.cpp +++ b/lib/microLB/micro_lb/autoconf.cpp @@ -23,8 +23,10 @@ namespace microLB assert(CLIENT_PORT > 0 && CLIENT_PORT < 65536); // client wait queue limit const int CLIENT_WAITQ = clients["waitq_limit"].GetUint(); + (void) CLIENT_WAITQ; // client session limit const int CLIENT_SLIMIT = clients["session_limit"].GetUint(); + (void) CLIENT_SLIMIT; auto& nodes = obj["nodes"]; const int NODE_NET = nodes["iface"].GetInt(); diff --git a/src/crt/cxx_abi.cpp b/src/crt/cxx_abi.cpp index e64100e89a..10bfd53443 100644 --- a/src/crt/cxx_abi.cpp +++ b/src/crt/cxx_abi.cpp @@ -65,7 +65,7 @@ extern "C" } using ub_error = std::runtime_error; - void undefined_throw(const char*) { + void undefined_throw(const char*) { } diff --git a/src/drivers/ide.cpp b/src/drivers/ide.cpp index 2469e2c556..d3ecdf18c3 100644 --- a/src/drivers/ide.cpp +++ b/src/drivers/ide.cpp @@ -181,7 +181,7 @@ IDE::IDE(hw::PCI_Device& pcidev, selector_t sel) { // 28-bits CHS (MAX_LBA) this->num_blocks = (read_array[61] << 16) | read_array[60]; } - INFO("IDE", "%u sectors (%lu bytes)", num_blocks, num_blocks * IDE::SECTOR_SIZE); + INFO("IDE", "%zu sectors (%zu bytes)", num_blocks, num_blocks * IDE::SECTOR_SIZE); INFO("IDE", "Initialization complete"); } diff --git a/src/drivers/solo5blk.cpp b/src/drivers/solo5blk.cpp index dceb4ad488..aad5a24d42 100644 --- a/src/drivers/solo5blk.cpp +++ b/src/drivers/solo5blk.cpp @@ -16,7 +16,7 @@ extern "C" { Solo5Blk::Solo5Blk() : hw::Block_device() { - INFO("Solo5Blk", "Block device with %llu sectors", solo5_blk_sectors()); + INFO("Solo5Blk", "Block device with %zu sectors", solo5_blk_sectors()); } Solo5Blk::block_t Solo5Blk::size() const noexcept { diff --git a/src/drivers/solo5blk.hpp b/src/drivers/solo5blk.hpp index fe14fa3f31..23b9eca0f3 100644 --- a/src/drivers/solo5blk.hpp +++ b/src/drivers/solo5blk.hpp @@ -77,9 +77,6 @@ class Solo5Blk : public hw::Block_device Solo5Blk(); private: - // stat counters - uint32_t* errors; - uint32_t* requests; }; #endif diff --git a/src/drivers/virtioblk.cpp b/src/drivers/virtioblk.cpp index 4a6b77017d..c4da500ffb 100644 --- a/src/drivers/virtioblk.cpp +++ b/src/drivers/virtioblk.cpp @@ -81,7 +81,7 @@ VirtioBlk::VirtioBlk(hw::PCI_Device& d) // Step 3 - Fill receive queue with buffers // DEBUG: Disable - INFO("VirtioBlk", "Queue size: %i\tRequest size: %u\n", + INFO("VirtioBlk", "Queue size: %i\tRequest size: %zu\n", req.size(), sizeof(request_t)); // Get device configuration @@ -107,7 +107,7 @@ VirtioBlk::VirtioBlk(hw::PCI_Device& d) } // Done - INFO("VirtioBlk", "Block device with %llu sectors capacity", config.capacity); + INFO("VirtioBlk", "Block device with %zu sectors capacity", config.capacity); } void VirtioBlk::get_config() @@ -254,7 +254,7 @@ void VirtioBlk::read (block_t blk, size_t cnt, on_read_func func) [this, i, func, results, bigbuf] (uint8_t* data) { // if the job was already completed, return early if (*results == 0) { - printf("Job cancelled? results == 0, blk=%u\n", i); + printf("[virtioblk] Job cancelled? results == 0, blk=%zu\n", i); return; } // validate partial result diff --git a/src/drivers/virtionet.cpp b/src/drivers/virtionet.cpp index 44f963a1fd..bd566cd2f7 100644 --- a/src/drivers/virtionet.cpp +++ b/src/drivers/virtionet.cpp @@ -60,7 +60,7 @@ void VirtioNet::get_config() { } #define VNET_TOT_BUFFERS() (48 + (queue_size(0) + queue_size(1)) / 2) -VirtioNet::VirtioNet(hw::PCI_Device& d, const uint16_t mtu) +VirtioNet::VirtioNet(hw::PCI_Device& d, const uint16_t /*mtu*/) : Virtio(d), Link(Link_protocol{{this, &VirtioNet::transmit}, mac()}, bufstore_), m_pcidev(d), diff --git a/src/kernel/context.cpp b/src/kernel/context.cpp index eafabb86a5..236a0b33db 100644 --- a/src/kernel/context.cpp +++ b/src/kernel/context.cpp @@ -19,7 +19,11 @@ #include #include +#ifdef ARCH_i686 extern "C" void __fastcall __context_switch(uintptr_t stack, Context::context_func& func); +#else +extern "C" void __context_switch(uintptr_t stack, Context::context_func& func); +#endif extern "C" void* get_cpu_esp(); extern "C" @@ -37,8 +41,8 @@ void Context::jump(void* location, context_func func) void Context::create(unsigned stack_size, context_func func) { // create stack, and jump to end (because it grows down) - auto stack_mem = - std::unique_ptr(new char[16+stack_size], + auto stack_mem = + std::unique_ptr(new char[16+stack_size], std::default_delete ()); jump(&(stack_mem.get()[stack_size]), func); } diff --git a/src/kernel/multiboot.cpp b/src/kernel/multiboot.cpp index 6900888109..b39277db1e 100644 --- a/src/kernel/multiboot.cpp +++ b/src/kernel/multiboot.cpp @@ -36,9 +36,14 @@ extern "C" uintptr_t _multiboot_free_begin(uintptr_t); using namespace util::bitops; using namespace util::literals; -multiboot_info_t* OS::bootinfo() +static inline multiboot_info_t* bootinfo(uint32_t addr) { // NOTE: the address is 32-bit and not a pointer + return (multiboot_info_t*) (uintptr_t) addr; +} + +multiboot_info_t* OS::bootinfo() +{ extern uint32_t __multiboot_addr; return (multiboot_info_t*) (uintptr_t) __multiboot_addr; } @@ -47,34 +52,34 @@ multiboot_info_t* OS::bootinfo() // (e.g. multiboot's data area as offset to the _end symbol) uintptr_t _multiboot_free_begin(uintptr_t boot_addr) { - multiboot_info_t* bootinfo = (multiboot_info_t*) boot_addr; + auto* info = bootinfo(boot_addr); uintptr_t multi_end = reinterpret_cast(&_end); - debug("* Multiboot begin: 0x%x \n", bootinfo); - if (bootinfo->flags & MULTIBOOT_INFO_CMDLINE - and bootinfo->cmdline > multi_end) + debug("* Multiboot begin: 0x%x \n", info); + if (info->flags & MULTIBOOT_INFO_CMDLINE + and info->cmdline > multi_end) { - debug("* Multiboot cmdline @ 0x%x: %s \n", bootinfo->cmdline, (char*)bootinfo->cmdline); + debug("* Multiboot cmdline @ 0x%x: %s \n", info->cmdline, (char*)info->cmdline); // We can't use a cmdline that's either insde our ELF or pre-ELF area - Expects(bootinfo->cmdline > multi_end - or bootinfo->cmdline < 0x100000); + Expects(info->cmdline > multi_end + or info->cmdline < 0x100000); - if (bootinfo->cmdline > multi_end) { - auto* cmdline_ptr = (const char*) (uintptr_t) bootinfo->cmdline; + if (info->cmdline > multi_end) { + auto* cmdline_ptr = (const char*) (uintptr_t) info->cmdline; // Set free begin to after the cmdline string - multi_end = bootinfo->cmdline + strlen(cmdline_ptr) + 1; + multi_end = info->cmdline + strlen(cmdline_ptr) + 1; } } debug("* Multiboot end: 0x%x \n", multi_end); - if (bootinfo->mods_count == 0) + if (info->mods_count == 0) return multi_end; - multiboot_module_t* mods_list = (multiboot_module_t*)bootinfo->mods_addr; + auto* mods_list = (multiboot_module_t*) (uintptr_t) info->mods_addr; debug("* Module list @ %p \n",mods_list); for (multiboot_module_t* mod = mods_list; - mod < mods_list + bootinfo->mods_count; + mod < mods_list + info->mods_count; mod ++) { debug("\t * Module @ %#x \n", mod->mod_start); @@ -93,16 +98,16 @@ uintptr_t _multiboot_free_begin(uintptr_t boot_addr) void OS::multiboot(uint32_t boot_addr) { MYINFO("Booted with multiboot"); - auto* bootinfo_ = bootinfo(); - INFO2("* Boot flags: %#x", bootinfo_->flags); + auto* info = ::bootinfo(boot_addr); + INFO2("* Boot flags: %#x", info->flags); - if (bootinfo_->flags & MULTIBOOT_INFO_MEMORY) { + if (info->flags & MULTIBOOT_INFO_MEMORY) { uint32_t mem_low_start = 0; - uint32_t mem_low_end = (bootinfo_->mem_lower * 1024) - 1; - uint32_t mem_low_kb = bootinfo_->mem_lower; + uint32_t mem_low_end = (info->mem_lower * 1024) - 1; + uint32_t mem_low_kb = info->mem_lower; uint32_t mem_high_start = 0x100000; - uint32_t mem_high_end = mem_high_start + (bootinfo_->mem_upper * 1024) - 1; - uint32_t mem_high_kb = bootinfo_->mem_upper; + uint32_t mem_high_end = mem_high_start + (info->mem_upper * 1024) - 1; + uint32_t mem_high_kb = info->mem_upper; OS::memory_end_ = mem_high_end; @@ -117,18 +122,19 @@ void OS::multiboot(uint32_t boot_addr) INFO2("No memory information from multiboot"); } - if (bootinfo_->flags & MULTIBOOT_INFO_CMDLINE) { - const auto* cmdline = (const char*) (uintptr_t) bootinfo_->cmdline; + if (info->flags & MULTIBOOT_INFO_CMDLINE) { + const auto* cmdline = (const char*) (uintptr_t) info->cmdline; INFO2("* Booted with parameters @ %p: %s", cmdline, cmdline); OS::cmdline = strdup(cmdline); } - if (bootinfo_->flags & MULTIBOOT_INFO_MEM_MAP) { - INFO2("* Multiboot provided memory map (%i entries @ %p)", - bootinfo_->mmap_length / sizeof(multiboot_memory_map_t), (void*)bootinfo_->mmap_addr); + if (info->flags & MULTIBOOT_INFO_MEM_MAP) { + INFO2("* Multiboot provided memory map (%zu entries @ %p)", + info->mmap_length / sizeof(multiboot_memory_map_t), + (void*) (uintptr_t) info->mmap_addr); gsl::span mmap { - reinterpret_cast(bootinfo_->mmap_addr), - (int)(bootinfo_->mmap_length / sizeof(multiboot_memory_map_t)) + reinterpret_cast(info->mmap_addr), + (int)(info->mmap_length / sizeof(multiboot_memory_map_t)) }; for (auto map : mmap) @@ -136,13 +142,13 @@ void OS::multiboot(uint32_t boot_addr) const char* str_type = map.type & MULTIBOOT_MEMORY_AVAILABLE ? "FREE" : "RESERVED"; const uintptr_t addr = map.addr; const uintptr_t size = map.len; - INFO2(" 0x%010llx - 0x%010llx %s (%llu Kb.)", + INFO2(" 0x%010zx - 0x%010zx %s (%zu Kb.)", addr, addr + size - 1, str_type, size / 1024 ); if (not (map.type & MULTIBOOT_MEMORY_AVAILABLE)) { if (util::bits::is_aligned<4_KiB>(map.addr)) { - os::mem::map({addr, addr, {os::mem::Access::read | os::mem::Access::write}, size}, + os::mem::map({addr, addr, os::mem::Access::read | os::mem::Access::write, size}, "Reserved (Multiboot)"); continue; } @@ -162,7 +168,7 @@ void OS::multiboot(uint32_t boot_addr) Span_mods mods = modules(); if (not mods.empty()) { - MYINFO("OS loaded with %i modules", mods.size()); + MYINFO("OS loaded with %zu modules", mods.size()); for (auto mod : mods) { INFO2("* %s @ 0x%x - 0x%x, size: %ib", reinterpret_cast(mod.cmdline), diff --git a/src/kernel/profile.cpp b/src/kernel/profile.cpp index 0cc616c32b..bf61427684 100644 --- a/src/kernel/profile.cpp +++ b/src/kernel/profile.cpp @@ -168,7 +168,7 @@ std::vector StackSampler::results(int N) res.push_back(Sample {sa.second, (void*) func.addr, func.name}); } else { - int len = snprintf(buffer, sizeof(buffer), "0x%08x", func.addr); + int len = snprintf(buffer, sizeof(buffer), "0x%08zx", func.addr); res.push_back(Sample {sa.second, (void*) func.addr, std::string(buffer, len)}); } @@ -182,14 +182,14 @@ void StackSampler::print(const int N) auto samp = results(N); int total = samples_total(); - printf("Stack sampling - %d results (%u samples)\n", + printf("Stack sampling - %zu results (%d samples)\n", samp.size(), total); for (auto& sa : samp) { // percentage of total samples float perc = sa.samp / (float)total * 100.0f; printf("%5.2f%% %*u: %.*s\n", - perc, 8, sa.samp, sa.name.size(), sa.name.c_str()); + perc, 8, sa.samp, (int) sa.name.size(), sa.name.c_str()); } } diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index 230c2ac11b..485ab8a495 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -20,7 +20,6 @@ #include // open() #include #include -#include #include #include #include diff --git a/src/musl/chmod.cpp b/src/musl/chmod.cpp index a794681461..a9162726ae 100644 --- a/src/musl/chmod.cpp +++ b/src/musl/chmod.cpp @@ -1,10 +1,9 @@ #include "common.hpp" #include -static int sys_chmod(const char *path, mode_t mode) { +static int sys_chmod(const char* /*path*/, mode_t /*mode*/) { // currently makes no sense, especially since we're read-only - errno = EROFS; - return -1; + return -EROFS; } extern "C" diff --git a/src/musl/creat.cpp b/src/musl/creat.cpp index 3785b0adb1..8e673ca88c 100644 --- a/src/musl/creat.cpp +++ b/src/musl/creat.cpp @@ -1,7 +1,7 @@ #include "stub.hpp" #include -static long sys_creat(const char *pathname, mode_t mode) { +static long sys_creat(const char* /*pathname*/, mode_t /*mode*/) { // currently makes no sense, especially since we're read-only return -EROFS; } diff --git a/src/musl/exit.cpp b/src/musl/exit.cpp index 862cf36d60..eaf759f110 100644 --- a/src/musl/exit.cpp +++ b/src/musl/exit.cpp @@ -3,13 +3,16 @@ #include #include -static int sys_exit(int status){ +__attribute__((noreturn)) +static long sys_exit(int status) +{ const std::string msg = "Service exited with status " + std::to_string(status) + "\n"; OS::print(msg.data(), msg.size()); __arch_poweroff(); + __builtin_unreachable(); } -extern "C" __attribute__((noreturn)) +extern "C" void syscall_SYS_exit(int status) { strace(sys_exit, "exit", status); } diff --git a/src/musl/fchmod.cpp b/src/musl/fchmod.cpp index 5ace900453..0d2d31eab6 100644 --- a/src/musl/fchmod.cpp +++ b/src/musl/fchmod.cpp @@ -1,9 +1,8 @@ #include "common.hpp" #include -static int sys_fchmod(int fildes, mode_t mode) { +static int sys_fchmod(int /*fd*/, mode_t /*mode*/) { // currently makes no sense, especially since we're read-only - errno = EROFS; return -EROFS; } diff --git a/src/musl/fchmodat.cpp b/src/musl/fchmodat.cpp index 881ed21506..1029550e68 100644 --- a/src/musl/fchmodat.cpp +++ b/src/musl/fchmodat.cpp @@ -2,9 +2,9 @@ #include #include -static int sys_fchmodat(int fd, const char *path, mode_t mode, int flag) { +static int sys_fchmodat(int /*fd*/, const char* /*path*/, mode_t, int /*flag*/) +{ // currently makes no sense, especially since we're read-only - errno = EROFS; return -EROFS; } diff --git a/src/musl/fcntl.cpp b/src/musl/fcntl.cpp index a3fce61335..7094fbd1c8 100644 --- a/src/musl/fcntl.cpp +++ b/src/musl/fcntl.cpp @@ -1,6 +1,6 @@ #include "common.hpp" -static int sys_fcntl(int fd, int cmd, ...){ +static int sys_fcntl(int /*fd*/, int /*cmd*/, ...){ return 0; } diff --git a/src/musl/futex.cpp b/src/musl/futex.cpp index 10534c6000..1b7c46559f 100644 --- a/src/musl/futex.cpp +++ b/src/musl/futex.cpp @@ -17,8 +17,8 @@ extern void print_backtrace(); -static int sys_futex(int *uaddr, int futex_op, int val, - const struct timespec *timeout, int val3) +static int sys_futex(int *uaddr, int /*futex_op*/, int val, + const struct timespec *timeout, int /*val3*/) { if (*uaddr != val){ diff --git a/src/musl/getdents.cpp b/src/musl/getdents.cpp index e8817916c0..8a280e4b18 100644 --- a/src/musl/getdents.cpp +++ b/src/musl/getdents.cpp @@ -1,9 +1,9 @@ #include "common.hpp" -static int sys_getdents(unsigned int fd, struct linux_dirent *dirp, unsigned int count) { +static int sys_getdents(int /*fd*/, struct linux_dirent*, unsigned int /*count*/) +{ // currently makes no sense, especially since we're read-only - errno = EROFS; - return -1; + return -EROFS; } extern "C" diff --git a/src/musl/getrlimit.cpp b/src/musl/getrlimit.cpp index ab8b6edef9..7db87424f9 100644 --- a/src/musl/getrlimit.cpp +++ b/src/musl/getrlimit.cpp @@ -2,9 +2,8 @@ #include "stub.hpp" -static int sys_getrlimit(int resource, struct rlimit *rlim) { - errno = ENOSYS; - return -1; +static int sys_getrlimit(int /*resource*/, struct rlimit*) { + return -ENOSYS; } extern "C" diff --git a/src/musl/ioctl.cpp b/src/musl/ioctl.cpp index 660b134e7c..13e387ffd2 100644 --- a/src/musl/ioctl.cpp +++ b/src/musl/ioctl.cpp @@ -13,7 +13,8 @@ static int sys_ioctl(int fd, int req, void* arg) { } if (req == TIOCSWINSZ) { - winsize* ws = (winsize*)arg; + const auto* ws = (winsize*) arg; + (void) ws; } return 0; diff --git a/src/musl/lseek.cpp b/src/musl/lseek.cpp index 1d975e32f7..d6b46eda66 100644 --- a/src/musl/lseek.cpp +++ b/src/musl/lseek.cpp @@ -4,14 +4,14 @@ #include "stub.hpp" -off_t sys_lseek(int fd, off_t offset, int whence) { +off_t sys_lseek(int /*fd*/, off_t /*offset*/, int /*whence*/) { errno = ENOSYS; return -1; } -off_t sys__llseek(unsigned int fd, unsigned long offset_high, - unsigned long offset_low, loff_t *result, - unsigned int whence) { +off_t sys__llseek(unsigned int /*fd*/, unsigned long /*offset_high*/, + unsigned long /*offset_low*/, loff_t* /*result*/, + unsigned int /*whence*/) { errno = ENOSYS; return -1; } diff --git a/src/musl/mkdir.cpp b/src/musl/mkdir.cpp index e3e9b7f724..33dbdc68ae 100644 --- a/src/musl/mkdir.cpp +++ b/src/musl/mkdir.cpp @@ -1,7 +1,7 @@ #include "common.hpp" #include -static long sys_mkdir(const char *path, mode_t mode) { +static long sys_mkdir(const char* /*path*/, mode_t /*mode*/) { // currently makes no sense, especially since we're read-only return -EROFS; } diff --git a/src/musl/mkdirat.cpp b/src/musl/mkdirat.cpp index b74278004f..0837276768 100644 --- a/src/musl/mkdirat.cpp +++ b/src/musl/mkdirat.cpp @@ -2,7 +2,7 @@ #include #include -static long sys_mkdirat(int fd, const char *path, mode_t mode) { +static long sys_mkdirat(int /*fd*/, const char* /*path*/, mode_t /*mode*/) { // currently makes no sense, especially since we're read-only return -EROFS; } diff --git a/src/musl/mknod.cpp b/src/musl/mknod.cpp index 255816113d..b1615be2a6 100644 --- a/src/musl/mknod.cpp +++ b/src/musl/mknod.cpp @@ -1,6 +1,7 @@ #include "common.hpp" -static long sys_mknod(const char *pathname, mode_t mode, dev_t dev) { +static long sys_mknod(const char* /*pathname*/, mode_t /*mode*/, dev_t /*dev*/) +{ // currently makes no sense, especially since we're read-only return -EROFS; } diff --git a/src/musl/mknodat.cpp b/src/musl/mknodat.cpp index 43e4f5b562..121dbbcbc0 100644 --- a/src/musl/mknodat.cpp +++ b/src/musl/mknodat.cpp @@ -1,6 +1,7 @@ #include "common.hpp" -static long sys_mknodat(int dirfd, const char *pathname, mode_t mode, dev_t dev) { +static long sys_mknodat(int /*dirfd*/, const char* /*path*/, mode_t, dev_t) +{ // currently makes no sense, especially since we're read-only return -EROFS; } diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 914d8ec6be..7c75eea0e7 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -6,34 +6,31 @@ #include extern uintptr_t heap_begin; -extern uintptr_t heap_end = 0; -static uintptr_t current_pos = 0; +extern uintptr_t heap_end; using Alloc = util::alloc::Lstack<4096>; static Alloc alloc; -void init_mmap(uintptr_t addr_begin){ +void init_mmap(uintptr_t addr_begin) +{ Expects(alloc.empty()); auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); alloc.donate((void*)aligned_begin, (OS::heap_max() - aligned_begin) & ~(Alloc::align - 1)); - } - extern "C" __attribute__((weak)) -void* __kalloc(size_t size){ +void* kalloc(size_t size) { return alloc.allocate(size); } extern "C" __attribute__((weak)) -void __kfree (void* ptr, size_t size){ +void kfree (void* ptr, size_t size) { alloc.deallocate(ptr, size); } -static void* sys_mmap(void *addr, size_t length, int prot, int flags, - int fd, off_t offset) +static void* sys_mmap(void *addr, size_t length, int /*prot*/, int /*flags*/, + int fd, off_t /*offset*/) { - // TODO: Mapping to file descriptor if (fd > 0) { assert(false && "Mapping to file descriptor not yet implemented"); @@ -41,11 +38,10 @@ static void* sys_mmap(void *addr, size_t length, int prot, int flags, // TODO: mapping virtual address if (addr) { - errno = ENODEV; return MAP_FAILED; } - return __kalloc(length);; + return kalloc(length); } extern "C" diff --git a/src/musl/open.cpp b/src/musl/open.cpp index 0b1b8254ef..efdd912334 100644 --- a/src/musl/open.cpp +++ b/src/musl/open.cpp @@ -1,9 +1,9 @@ #include "common.hpp" #include - #include -static long sys_open(const char *pathname, int flags, mode_t mode = 0) { +static long sys_open(const char *pathname, int /*flags*/, mode_t /*mode = 0*/) +{ try { auto& entry = fs::VFS::get(pathname); auto& fd = entry.open_fd(); diff --git a/src/musl/openat.cpp b/src/musl/openat.cpp index f190ead14b..3707331f6f 100644 --- a/src/musl/openat.cpp +++ b/src/musl/openat.cpp @@ -2,9 +2,9 @@ #include #include -static int sys_openat(int dirfd, const char *pathname, int flags, mode_t mode) { - errno = ENOSYS; - return -1; +static int sys_openat(int /*dirfd*/, const char* /*path*/, int /*flags*/, mode_t) +{ + return -ENOSYS; } extern "C" diff --git a/src/musl/poll.cpp b/src/musl/poll.cpp index 56901ad0dd..25070a356f 100644 --- a/src/musl/poll.cpp +++ b/src/musl/poll.cpp @@ -1,13 +1,11 @@ #include "common.hpp" #include -#warning stub - extern "C" -long syscall_SYS_poll(struct pollfd *fds, nfds_t nfds, int timeout) +long syscall_SYS_poll(struct pollfd *fds, nfds_t nfds, int /*timeout*/) { STUB("poll"); - for (int i = 0; i < nfds; i++) + for (nfds_t i = 0; i < nfds; i++) { fds[i].revents = fds[i].events; } diff --git a/src/musl/readv.cpp b/src/musl/readv.cpp index ccab1d6638..f5782f7f27 100644 --- a/src/musl/readv.cpp +++ b/src/musl/readv.cpp @@ -2,10 +2,9 @@ #include #include -static ssize_t sys_readv(int fd, const struct iovec *iov, int iovcnt) +static ssize_t sys_readv(int /*fd*/, const struct iovec*, int /*iovcnt*/) { - errno = ENOSYS; - return -1; + return -ENOSYS; } extern "C" diff --git a/src/musl/set_tid_address.cpp b/src/musl/set_tid_address.cpp index 206a4379aa..9f83daa68f 100644 --- a/src/musl/set_tid_address.cpp +++ b/src/musl/set_tid_address.cpp @@ -1,7 +1,5 @@ #include "stub.hpp" -#warning stub - static struct { int tid = 1; int* set_child_tid = nullptr; diff --git a/src/musl/sigmask.cpp b/src/musl/sigmask.cpp index 1aa4c210dd..b5da2816de 100644 --- a/src/musl/sigmask.cpp +++ b/src/musl/sigmask.cpp @@ -1,11 +1,11 @@ #include "stub.hpp" #include -static int sys_rt_sigprocmask (int how, const sigset_t *set, sigset_t *oldset){ +static int sys_rt_sigprocmask (int /*how*/, const sigset_t*, sigset_t* /*old*/){ return 0; } -static int sys_sigmask(int signum) +static int sys_sigmask(int /*signum*/) { return 0; } @@ -15,28 +15,27 @@ extern const bool __strace; extern "C" { int syscall_SYS_rt_sigprocmask(int how, const sigset_t *set, sigset_t *oldset) { - if constexpr (__strace) { + if constexpr (__strace) + { + const char* howstr = nullptr; - char* howstr = nullptr; + switch(how) { + case SIG_BLOCK: + howstr = "BLOCK"; + break; - switch(how) { + case SIG_UNBLOCK: + howstr = "UNBLOCK"; + break; - case (SIG_BLOCK): - howstr="BLOCK"; - break; - - case (SIG_UNBLOCK): - howstr="UNBLOCK"; - break; - - case (SIG_SETMASK): - howstr="SETMASK"; - break; - } - auto ret = sys_rt_sigprocmask(how, set, oldset); - stubtrace_print("sys_rt_sigprocmask", howstr, set, oldset); - return ret; + case SIG_SETMASK: + howstr = "SETMASK"; + break; } + auto ret = sys_rt_sigprocmask(how, set, oldset); + stubtrace_print("sys_rt_sigprocmask", howstr, set, oldset); + return ret; + } return sys_rt_sigprocmask(how, set, oldset); } diff --git a/src/musl/syscall_n.cpp b/src/musl/syscall_n.cpp index 8f569b1430..72cf3f24fb 100644 --- a/src/musl/syscall_n.cpp +++ b/src/musl/syscall_n.cpp @@ -1,8 +1,9 @@ #include "stub.hpp" -long syscall(long i){ - return 0; -}; +long syscall(long /*number*/) +{ + return -ENOSYS; +} extern "C" long syscall_n(long i) { diff --git a/src/musl/tkill.cpp b/src/musl/tkill.cpp index 3a20927cfe..0ecba0b550 100644 --- a/src/musl/tkill.cpp +++ b/src/musl/tkill.cpp @@ -1,12 +1,11 @@ #include "stub.hpp" #include -static int sys_tkill(int tid, int sig) { - +static int sys_tkill(int /*tid*/, int sig) +{ #ifndef INCLUDEOS_SINGLE_THREADED -#warning "tkill not implemented for threaded IncludeOS" +# warning "tkill not implemented for threaded IncludeOS" #endif - exit(sig); } diff --git a/src/musl/utimensat.cpp b/src/musl/utimensat.cpp index d35b6a03c6..e1fd81f0c1 100644 --- a/src/musl/utimensat.cpp +++ b/src/musl/utimensat.cpp @@ -3,26 +3,24 @@ #include #include -static int sys_utimensat(int dirfd, const char *pathname, - const struct timespec times[2], int flags) +static int sys_utimensat(int /*dirfd*/, const char* /*path*/, + const struct timespec[2], int /*flags*/) { // currently makes no sense, especially since we're read-only - errno = EROFS; - return -1; + return -EROFS; } // Obsolete, use utimensat -static int sys_futimesat(int dirfd, const char *pathname, - const struct timespec times[2]) +static int sys_futimesat(int dirfd, const char* path, + const struct timespec times[2]) { - return sys_utimensat(dirfd, pathname, times, 0); + return sys_utimensat(dirfd, path, times, 0); } -static int sys_utimes(const char *pathname, const struct timeval times[2]) +static int sys_utimes(const char* /*path*/, const struct timeval[2]) { // currently makes no sense, especially since we're read-only - errno = EROFS; - return -1; + return -EROFS; } extern "C" diff --git a/src/net/buffer_store.cpp b/src/net/buffer_store.cpp index 180f5097ed..66bcf16e6b 100644 --- a/src/net/buffer_store.cpp +++ b/src/net/buffer_store.cpp @@ -30,11 +30,8 @@ extern void *memalign(size_t, size_t); //#define DEBUG_RELEASE //#define DEBUG_RETRIEVE //#define DEBUG_BUFSTORE - -#define PAGE_SIZE 0x1000 #define ENABLE_BUFFERSTORE_CHAIN - #ifdef DEBUG_RELEASE #define BSD_RELEASE(fmt, ...) printf(fmt, ##__VA_ARGS__); #else diff --git a/src/net/ip4/icmp4.cpp b/src/net/ip4/icmp4.cpp index f8fc707a1a..688f37041b 100644 --- a/src/net/ip4/icmp4.cpp +++ b/src/net/ip4/icmp4.cpp @@ -255,7 +255,8 @@ namespace net { icmp4::Packet res(inet_.ip_packet_factory()); // drop if the packet is too small - if (res.ip().capacity() < res.ip().ip_header_length() + res.header_size() + req.payload().size()) + if (res.ip().capacity() < res.ip().ip_header_length() + + (int) res.header_size() + req.payload().size()) { printf("WARNING: Network MTU too small for ICMP response, dropping\n"); return; diff --git a/src/net/ip4/ip4.cpp b/src/net/ip4/ip4.cpp index 439761a868..b91d58eace 100644 --- a/src/net/ip4/ip4.cpp +++ b/src/net/ip4/ip4.cpp @@ -104,7 +104,7 @@ namespace net { or local_ip() == ADDR_ANY; } - void IP4::receive(Packet_ptr pckt, const bool link_bcast) + void IP4::receive(Packet_ptr pckt, const bool /*link_bcast*/) { // Cast to IP4 Packet auto packet = static_unique_ptr_cast(std::move(pckt)); diff --git a/src/net/openssl/init.cpp b/src/net/openssl/init.cpp index 3326b2df4a..cd6e6a4a81 100644 --- a/src/net/openssl/init.cpp +++ b/src/net/openssl/init.cpp @@ -51,7 +51,6 @@ namespace openssl } void verify_rng() { - auto* rm = RAND_get_rand_method(); int random_value = 0; int rc = RAND_bytes((uint8_t*) &random_value, sizeof(random_value)); assert(rc == 0 || rc == 1); diff --git a/src/platform/x86_pc/smp.cpp b/src/platform/x86_pc/smp.cpp index 5e94bb86a8..f5fc5f659f 100644 --- a/src/platform/x86_pc/smp.cpp +++ b/src/platform/x86_pc/smp.cpp @@ -216,6 +216,8 @@ void SMP::signal(int cpu) // 1-xx: Unicast specific vCPU else x86::APIC::get().send_ipi(cpu, 0x20); +#else + (void) cpu; #endif } void SMP::signal_bsp() diff --git a/src/plugins/unik.cpp b/src/plugins/unik.cpp index e9c135eef3..7b46c4381c 100644 --- a/src/plugins/unik.cpp +++ b/src/plugins/unik.cpp @@ -36,7 +36,7 @@ void unik::Client::register_instance(net::Inet& inet, const net::UDP:: // Set up an UDP port for receiving UniK heartbeat auto& sock = inet.udp().bind(port); CHECK(net::Inet4::stack<0>().udp().is_bound(sock.local()), "Unik UDP port is bound as expected"); - sock.on_read([&sock, &inet] (auto addr, auto port, const char* data, size_t len) { + sock.on_read([&inet] (auto addr, auto port, const char* data, size_t len) { static bool registered_with_unik = false; static const int max_attempts = 5; diff --git a/src/posix/fd.cpp b/src/posix/fd.cpp index c1cf812d70..8a9bac7ce2 100644 --- a/src/posix/fd.cpp +++ b/src/posix/fd.cpp @@ -46,20 +46,20 @@ int FD::fcntl(int cmd, va_list list) } } -int FD::ioctl(int req, void* arg) +int FD::ioctl(int /*req*/, void* /*arg*/) { //PRINT("ioctl(%d, %p) = -1\n", req, arg); errno = ENOSYS; return -1; } -int FD::getsockopt(int fd, int, void *__restrict__, socklen_t *__restrict__) +int FD::getsockopt(int /*fd*/, int, void *__restrict__, socklen_t *__restrict__) { //PRINT("getsockopt(%d) = -1\n", fd); errno = ENOTSOCK; return -1; } -int FD::setsockopt(int fd, int, const void *, socklen_t) +int FD::setsockopt(int /*fd*/, int, const void *, socklen_t) { //PRINT("setsockopt(%d) = -1\n", fd); errno = ENOTSOCK; diff --git a/src/util/async.cpp b/src/util/async.cpp index eb4be571c2..f83efa443e 100644 --- a/src/util/async.cpp +++ b/src/util/async.cpp @@ -81,7 +81,7 @@ void Async::disk_transfer( pos * CHUNK_SIZE, CHUNK_SIZE, fs::on_read_func::make_packed( - [next, pos, write_func, callback, CHUNK_SIZE] ( + [next, pos, write_func, callback] ( fs::error_t err, fs::buffer_t buffer) { From 913057c7af6ba19b3809c4adac09ddd33f3c5640 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 22 Feb 2018 18:44:27 +0100 Subject: [PATCH 088/723] Resolve cherry-pick conflict --- src/crt/c_abi.c | 101 +-------------------------- src/kernel/syscalls.cpp | 75 +++----------------- src/musl/CMakeLists.txt | 2 +- src/musl/exit_group.cpp | 7 -- src/platform/x86_pc/kernel_start.cpp | 2 + 5 files changed, 16 insertions(+), 171 deletions(-) delete mode 100644 src/musl/exit_group.cpp diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index 7643181577..f60fcd9a36 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -16,19 +16,14 @@ // limitations under the License. #include -#include -#include -#include +#include #include -#include -#include #include #define HEAP_ALIGNMENT 63 void* heap_begin; void* heap_end; - -void* __dso_handle; +void* __dso_handle = (void*) &__dso_handle; // stack-protector guard const uintptr_t __stack_chk_guard = (uintptr_t) _STACK_GUARD_VALUE_; @@ -61,100 +56,10 @@ static void crt_sanity_checks() assert(validate_heap_alignment); } - -void _init_c_runtime() -{ - - /// init ELF / backtrace functionality - extern void _init_elf_parser(); - _init_elf_parser(); - kprintf("Elf parser initialized \n"); - crt_sanity_checks(); - kprintf("Sanity checks OK \n"); -} - - // stack-protector -__attribute__((noreturn)) +__attribute__((noreturn, used)) void __stack_chk_fail(void) { panic("Stack protector: Canary modified"); __builtin_unreachable(); } - -__attribute__((noreturn)) -void __stack_chk_fail_local(void) -{ - panic("Stack protector: Canary modified"); - __builtin_unreachable(); -} - -// old function result system -int errno = 0; -int* __errno_location(void) -{ - return &errno; -} - - -// linux strchr variant (NOTE: not completely the same!) -void *__rawmemchr (const void *s, int c) -{ - return strchr((const char*) s, c); -} - - -/// sched_yield() causes the calling thread to relinquish the CPU. The -/// thread is moved to the end of the queue for its static priority -/// and a new thread gets to run. -int sched_yield(void) -{ - // On success, sched_yield() returns 0. On error, -1 is returned, and - // errno is set appropriately. - //errno = ...; - return -1; -} - -void* __memcpy_chk(void* dest, const void* src, size_t len, size_t destlen) -{ - assert (len <= destlen); - return memcpy(dest, src, len); -} -void * __memset_chk(void* dest, int c, size_t len, size_t destlen) -{ - assert (len <= destlen); - return memset(dest, c, len); -} - -void *aligned_alloc(size_t alignment, size_t size) -{ - return memalign(alignment, size); -} - -int access(const char *pathname, int mode) -{ - (void) pathname; - (void) mode; - - return 0; -} - -int fcntl(int fd, int cmd, ...) -{ - (void) fd; - (void) cmd; - return 0; -} - -int rmdir(const char *pathname) -{ - (void) pathname; - return 0; -} - -int settimeofday(const struct timeval *tv, const struct timezone *tz) -{ - (void) tv; - (void) tz; - return 0; -} diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index 485ab8a495..207bd140d8 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -16,18 +16,13 @@ // limitations under the License. #include - -#include // open() -#include -#include -#include #include #include - #include #include #include #include +#include #if defined (UNITTESTS) && !defined(__MACH__) #define THROW throw() @@ -39,48 +34,22 @@ #define SYSINFO(TEXT, ...) kprintf("%13s ] " TEXT "\n", "[ Kernel", ##__VA_ARGS__) // Emitted if and only if panic (unrecoverable system wide error) happens -const char* panic_signature = "\x15\x07\t**** PANIC ****"; - -char* __env[1] {nullptr}; -char** environ {__env}; - +static const char* panic_signature = "\x15\x07\t**** PANIC ****"; extern uintptr_t heap_begin; extern uintptr_t heap_end; - void _exit(int status) { - kprintf("%s",std::string(LINEWIDTH, '=').c_str()); - kprint("\n"); - SYSINFO("service exited with status %i", status); + SYSINFO("Service exiting with status %d", status); default_exit(); __builtin_unreachable(); } -clock_t times(struct tms*) { - panic("SYSCALL TIMES Dummy, returning -1"); - return -1; -} - -int wait(int*) { - debug((char*)"SYSCALL WAIT Dummy, returning -1"); - return -1; -} - - -int kill(pid_t pid, int sig) THROW { - SMP::global_lock(); - printf("!!! Kill PID: %i, SIG: %i - %s ", pid, sig, strsignal(sig)); - - if (sig == 6) { - printf("/ ABORT\n"); - } else { - printf("\n"); - } - SMP::global_unlock(); - - panic("kill called"); - errno = ESRCH; - return -1; +extern "C" +void syscall_SYS_exit_group(int status) +{ + SYSINFO("Service exiting with status %d", status); + default_exit(); + __builtin_unreachable(); } struct alignas(SMP_ALIGN) context_buffer @@ -224,31 +193,7 @@ void default_exit() { __builtin_unreachable(); } -#if defined(__MACH__) -#if !defined(__MAC_10_12) -typedef int clockid_t; -#endif -#if !defined(CLOCK_REALTIME) -#define CLOCK_REALTIME 0 -#endif -#endif - -int clock_gettime(clockid_t clk_id, struct timespec* tp) { - if (clk_id == CLOCK_REALTIME) { - *tp = __arch_wall_clock(); - return 0; - } - printf("hmm clock_gettime called, -1\n"); - return -1; -} -int gettimeofday(struct timeval* p, void*) { - auto tval = __arch_wall_clock(); - p->tv_sec = tval.tv_sec; - p->tv_usec = tval.tv_nsec / 1000; - return 0; -} - -extern "C" void _init_syscalls(); +extern "C" void _init_syscalls() { // make sure each buffer is zero length so it won't always show up in crashes diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index 094d6a01da..2edcfa79a6 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -5,7 +5,7 @@ set(MUSL_OBJECTS lseek.cpp sched_getaffinity.cpp sysinfo.cpp prlimit64.cpp getrlimit.cpp sched_yield.cpp set_robust_list.cpp nanosleep.cpp open.cpp creat.cpp clock_gettime.cpp gettimeofday.cpp - poll.cpp exit_group.cpp exit.cpp close.cpp set_tid_address.cpp + poll.cpp exit.cpp close.cpp set_tid_address.cpp pipe.cpp read.cpp readv.cpp getpid.cpp mknod.cpp sync.cpp msync.cpp mincore.cpp syscall_n.cpp sigmask.cpp gettid.cpp tkill.cpp stat.cpp fstat.cpp fstatat.cpp diff --git a/src/musl/exit_group.cpp b/src/musl/exit_group.cpp deleted file mode 100644 index b9f5065a6f..0000000000 --- a/src/musl/exit_group.cpp +++ /dev/null @@ -1,7 +0,0 @@ -#include "common.hpp" - -extern "C" -long syscall_SYS_exit_group() { - STUB("exit_group"); - return 0; -} diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 5b9c9d4f89..dd98cc455e 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -146,6 +146,7 @@ void kernel_start(uintptr_t magic, uintptr_t addr) // Initialize CPU exceptions x86::idt_initialize_for_cpu(0); + kprintf("* Thread local1: %i\n", __tl1__); kprintf("* Elf start: %p\n", &_ELF_START_); @@ -164,6 +165,7 @@ void kernel_start(uintptr_t magic, uintptr_t addr) kprintf("\tPhdr %i @ %p, va_addr: 0x%lx \n", i, &phdr[i], phdr[i].p_vaddr); } + // Build AUX-vector for C-runtime auxv_t aux[38]; kprintf("* Initializing aux-vector @ %p\n", aux); From e1e593a08d5911e009c2ce7017f025238c6d7a4b Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Sun, 18 Mar 2018 20:59:13 +0100 Subject: [PATCH 089/723] musl: __kfree to kfree in munmap syscall --- src/musl/munmap.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/musl/munmap.cpp b/src/musl/munmap.cpp index a8ff4e172a..5db7ca4bb5 100644 --- a/src/musl/munmap.cpp +++ b/src/musl/munmap.cpp @@ -1,12 +1,11 @@ #include "common.hpp" -extern "C" -void __kfree(void* addr, size_t length); +extern "C" void kfree(void* addr, size_t length); extern "C" int syscall_SYS_munmap(void *addr, size_t length) { STUB("munmap"); - __kfree(addr, length); + kfree(addr, length); return 0; } From d1a6eb2825c83929c07e4ad6f730ca230fbe9cb6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Sun, 18 Mar 2018 21:08:27 +0100 Subject: [PATCH 090/723] crt: Remove unused stuff in c_abi.c --- src/crt/c_abi.c | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index f60fcd9a36..c2efaaab68 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -17,17 +17,8 @@ #include #include -#include #include - #define HEAP_ALIGNMENT 63 -void* heap_begin; -void* heap_end; -void* __dso_handle = (void*) &__dso_handle; - -// stack-protector guard -const uintptr_t __stack_chk_guard = (uintptr_t) _STACK_GUARD_VALUE_; -extern void panic(const char* why) __attribute__((noreturn)); uint32_t _move_symbols(void* sym_loc) { @@ -43,23 +34,3 @@ uint32_t _move_symbols(void* sym_loc) return elfsym_size; } - -static void crt_sanity_checks() -{ - // validate that heap is aligned - int validate_heap_alignment = - ((uintptr_t)heap_begin & (uintptr_t) HEAP_ALIGNMENT) == 0; - - extern char _end; - assert(heap_begin >= (void*) &_end); - assert(heap_end >= heap_begin); - assert(validate_heap_alignment); -} - -// stack-protector -__attribute__((noreturn, used)) -void __stack_chk_fail(void) -{ - panic("Stack protector: Canary modified"); - __builtin_unreachable(); -} From e6dbc17299da81bee6619124187681017734e02d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 20 Mar 2018 11:57:37 +0100 Subject: [PATCH 091/723] musl: Work on syscalls, specifically files and getdents --- api/posix/fd.hpp | 3 + api/posix/file_fd.hpp | 2 + src/musl/cwd.cpp | 12 +- src/musl/exit_group.cpp | 18 ++- src/musl/fcntl.cpp | 18 ++- src/musl/getdents.cpp | 15 ++- src/musl/open.cpp | 6 +- src/musl/stat.cpp | 1 + src/musl/write.cpp | 3 +- src/musl/writev.cpp | 3 +- src/posix/file_fd.cpp | 124 +++++++++++++++++- .../integration/file_fd/test_file_fd.cpp | 1 + test/posix/integration/stat/ftw_tests.cpp | 4 +- 13 files changed, 183 insertions(+), 27 deletions(-) diff --git a/api/posix/fd.hpp b/api/posix/fd.hpp index deb838ed8f..6cab026e4e 100644 --- a/api/posix/fd.hpp +++ b/api/posix/fd.hpp @@ -71,6 +71,9 @@ class FD { virtual int mknodat(const char *, mode_t, dev_t) { return -1; } virtual int lseek(off_t, int) { return -1; } + // linux specific + virtual long getdents([[maybe_unused]]struct dirent *dirp, [[maybe_unused]]unsigned int count) { return -1; } + id_t get_id() const noexcept { return id_; } virtual bool is_file() { return false; } diff --git a/api/posix/file_fd.hpp b/api/posix/file_fd.hpp index f7ea47311e..b2e5ff4495 100644 --- a/api/posix/file_fd.hpp +++ b/api/posix/file_fd.hpp @@ -32,6 +32,8 @@ class File_FD : public FD { int close() override; int lseek(off_t, int) override; + long getdents(struct dirent *dirp, unsigned int count) override; + bool is_file() override { return true; } private: diff --git a/src/musl/cwd.cpp b/src/musl/cwd.cpp index d3f945a1e9..6a72d03a0a 100644 --- a/src/musl/cwd.cpp +++ b/src/musl/cwd.cpp @@ -2,20 +2,22 @@ #include #include -#include + static std::string cwd{"/"}; static long sys_chdir(const char* path) { // todo: handle relative path // todo: handle .. - if (UNLIKELY(not path or strlen(path) < 1)) + if (UNLIKELY(path == nullptr)) + return -ENOENT; + + if (UNLIKELY(strlen(path) < 1)) return -ENOENT; if (strcmp(path, ".") == 0) - { return 0; - } + std::string desired_path; if (*path != '/') { @@ -34,7 +36,6 @@ static long sys_chdir(const char* path) cwd = desired_path; assert(cwd.front() == '/'); assert(cwd.find("..") == std::string::npos); - printf("cwd: %s\n", cwd.c_str()); return 0; } else @@ -69,6 +70,7 @@ long sys_getcwd(char *buf, size_t size) extern "C" long syscall_SYS_chdir(const char* path) { return strace(sys_chdir, "chdir", path); + //return sys_chdir(path); } extern "C" diff --git a/src/musl/exit_group.cpp b/src/musl/exit_group.cpp index b9f5065a6f..ca8f65df37 100644 --- a/src/musl/exit_group.cpp +++ b/src/musl/exit_group.cpp @@ -1,7 +1,17 @@ #include "common.hpp" +#include +#include +#include -extern "C" -long syscall_SYS_exit_group() { - STUB("exit_group"); - return 0; +static long sys_exit_group(int status) { + const std::string msg = "(Exit group - terminating threads) Service exited with status " + + std::to_string(status) + "\n"; + OS::print(msg.data(), msg.size()); + __arch_poweroff(); + return status; +} + +extern "C" __attribute__((noreturn)) +long syscall_SYS_exit_group(int status) { + strace(sys_exit_group, "exit_group", status); } diff --git a/src/musl/fcntl.cpp b/src/musl/fcntl.cpp index a3fce61335..0bf1c59b43 100644 --- a/src/musl/fcntl.cpp +++ b/src/musl/fcntl.cpp @@ -1,11 +1,21 @@ #include "common.hpp" +#include -static int sys_fcntl(int fd, int cmd, ...){ - return 0; +static long sys_fcntl(int fd, int cmd, va_list va) { + try { + return FD_map::_get(fd).fcntl(cmd, va); + } + catch(const FD_not_found&) { + return -EBADF; + } } extern "C" -int syscall_SYS_fcntl(int fd, int cmd, ... /* arg */ ) +long syscall_SYS_fcntl(int fd, int cmd, ... /* arg */ ) { - return strace(sys_fcntl, "fcntl", fd, cmd); + va_list va; + va_start(va, cmd); + auto ret = strace(sys_fcntl, "fcntl", fd, cmd, va); + va_end(va); + return ret; } diff --git a/src/musl/getdents.cpp b/src/musl/getdents.cpp index e8817916c0..821cd86d94 100644 --- a/src/musl/getdents.cpp +++ b/src/musl/getdents.cpp @@ -1,12 +1,17 @@ #include "common.hpp" +#include +#include -static int sys_getdents(unsigned int fd, struct linux_dirent *dirp, unsigned int count) { - // currently makes no sense, especially since we're read-only - errno = EROFS; - return -1; +static long sys_getdents(unsigned int fd, struct dirent *dirp, unsigned int count) { + try { + return FD_map::_get(fd).getdents(dirp, count); + } + catch(const FD_not_found&) { + return -EBADF; + } } extern "C" -int syscall_SYS_getdents(unsigned int fd, struct linux_dirent *dirp, unsigned int count) { +long syscall_SYS_getdents(unsigned int fd, struct dirent *dirp, unsigned int count) { return strace(sys_getdents, "getdents", fd, dirp, count); } diff --git a/src/musl/open.cpp b/src/musl/open.cpp index 9570128c47..67d52ff743 100644 --- a/src/musl/open.cpp +++ b/src/musl/open.cpp @@ -18,7 +18,7 @@ static long sys_open(const char *pathname, int flags, mode_t mode = 0) { return fd.get_id(); } // Not FD_compatible, try dirent - catch(const fs::Err_bad_cast&) + catch(const fs::VFS_err& err) { try { auto ent = fs::VFS::stat_sync(pathname); @@ -34,10 +34,6 @@ static long sys_open(const char *pathname, int flags, mode_t mode = 0) { return -ENOENT; } } - catch(const fs::VFS_err&) - { - // VFS error for one of many reasons (not mounted, not fd compatible etc) - } catch(const std::bad_function_call&) { // Open fd delegate not set } diff --git a/src/musl/stat.cpp b/src/musl/stat.cpp index 2f5cbab09d..39c965564d 100644 --- a/src/musl/stat.cpp +++ b/src/musl/stat.cpp @@ -28,6 +28,7 @@ long sys_stat(const char *path, struct stat *buf) buf->st_mtime = ent.modified(); buf->st_blocks = buf->st_size > 0 ? util::bits::roundto<512>(buf->st_size) : 0; buf->st_blksize = ent.fs().block_size(); + //printf("Is file? %s\n", S_ISREG(buf->st_mode) ? "YES" : "NO"); //PRINT("stat(%s, %p) == %d\n", path, buf, 0); return 0; } diff --git a/src/musl/write.cpp b/src/musl/write.cpp index 7922885c59..bc4601d48d 100644 --- a/src/musl/write.cpp +++ b/src/musl/write.cpp @@ -24,5 +24,6 @@ static long sys_write(int fd, char* str, size_t len) { // The syscall wrapper, using strace if enabled extern "C" long syscall_SYS_write(int fd, char* str, size_t len) { - return strace(sys_write, "write", fd, str, len); + //return strace(sys_write, "write", fd, str, len); + return sys_write(fd, str, len); } diff --git a/src/musl/writev.cpp b/src/musl/writev.cpp index 01f69bb6e8..c9ffb77b50 100644 --- a/src/musl/writev.cpp +++ b/src/musl/writev.cpp @@ -22,5 +22,6 @@ static ssize_t sys_writev(int fd, const struct iovec *iov, int iovcnt) extern "C" ssize_t syscall_SYS_writev(int fd, const struct iovec *iov, int iovcnt){ - return strace(sys_writev, "writev", fd, iov, iovcnt); + //return strace(sys_writev, "writev", fd, iov, iovcnt); + return sys_writev(fd, iov, iovcnt); } diff --git a/src/posix/file_fd.cpp b/src/posix/file_fd.cpp index 342cb6e6a4..2a6c77a884 100644 --- a/src/posix/file_fd.cpp +++ b/src/posix/file_fd.cpp @@ -15,8 +15,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include +#include int File_FD::read(void* p, size_t n) { auto buf = ent_.read(offset_, n); @@ -56,3 +57,124 @@ int File_FD::lseek(off_t offset, int whence) //PRINT("lseek(%lu, %d) == %d\n", offset, whence, offset_); return offset_; } + + +//struct linux_dirent { +// unsigned long d_ino; /* Inode number */ +// unsigned long d_off; /* Offset to next linux_dirent */ +// unsigned short d_reclen; /* Length of this linux_dirent */ +// char d_name[]; /* Filename (null-terminated) */ +// /* length is actually (d_reclen - 2 - +// offsetof(struct linux_dirent, d_name)) */ +// /* +// char pad; // Zero padding byte +// char d_type; // File type (only since Linux +// // 2.6.4); offset is (d_reclen - 1) +// */ +//} + + + +/* + DT_BLK This is a block device. + + DT_CHR This is a character device. + + DT_DIR This is a directory. + + DT_FIFO This is a named pipe (FIFO). + + DT_LNK This is a symbolic link. + + DT_REG This is a regular file. + + DT_SOCK This is a UNIX domain socket. +*/ +/*struct dirent +{ + ino_t d_ino; + off_t d_off; + unsigned short d_reclen; + unsigned char d_type; + char d_name[256]; +};*/ +struct __dirstream +{ + int fd; + off_t tell; + int buf_pos; + int buf_end; + //volatile int lock[1]; + char buf[2048]; +}; +long File_FD::getdents(struct dirent *dirp, unsigned int count) +{ + static_assert(NAME_MAX+1 == sizeof(dirent::d_name)); + printf("getdents(%p, %u) fd=%i \"%s\" (%s)\n", dirp, count, get_id(), ent_.name().c_str(), ent_.type_string().c_str()); + + auto* dir = (__dirstream*)(((char*)dirp) - sizeof(__dirstream) + sizeof(__dirstream::buf)); + //printf("dirp %p dir %p buf %p\n", dirp, dir, dir->buf); + //assert((void*)dir->buf == (void*)dirp); + + if(not ent_.is_dir()) + return -ENOTDIR; + + unsigned int written = 0; + auto list = ent_.ls(); + + auto* ptr = (char*)dirp; + //ptr -= 4; + for(auto& ent : list) + { + dirp = (dirent*) ptr; + //if(ent.name() == "." or ent.name() == "..") + // continue; + // inode number + dirp->d_ino = ent.block(); + + // file type + switch(ent.type()) + { + case fs::DIR: + dirp->d_type = DT_DIR; + break; + default: + dirp->d_type = DT_REG; + } + // offset to next dirent + dirp->d_off = 0; + + // filename + auto name = ent.name(); + const auto namelen = std::min(sizeof(dirp->d_name)-1, name.length()); + + // Length of this dirent + dirp->d_reclen = sizeof(dirent) - sizeof(dirp->d_name) + namelen + 1; + + written += dirp->d_reclen; + + if(written > count) + return -EINVAL; + + //std::memcpy(dirp->d_name, name.data(), namelen); + //dirp->d_name[namelen] = '\0'; + std::strncpy(dirp->d_name, name.data(), namelen); + + ptr += dirp->d_reclen; + } + //printf("dir %p fd=%u sizeof(buf)=%zu %p\n", dir, dir->fd, sizeof(dir->buf), dir->buf); + + dir->buf_end = written; + dir->buf_pos = 0; + + while(dir->buf_pos < dir->buf_end) + { + auto* de = (struct dirent*)(dir->buf + dir->buf_pos); + printf("ino: %zu name: %s type: %u len: %u\n", de->d_ino, de->d_name, de->d_type, de->d_reclen); + if(de->d_reclen == 0) break; + dir->buf_pos += de->d_reclen; + dir->tell = de->d_off; + } + + return written; +} diff --git a/test/posix/integration/file_fd/test_file_fd.cpp b/test/posix/integration/file_fd/test_file_fd.cpp index c2a457b8cd..7748a63306 100644 --- a/test/posix/integration/file_fd/test_file_fd.cpp +++ b/test/posix/integration/file_fd/test_file_fd.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include diff --git a/test/posix/integration/stat/ftw_tests.cpp b/test/posix/integration/stat/ftw_tests.cpp index 51d6aec421..d6880cd945 100644 --- a/test/posix/integration/stat/ftw_tests.cpp +++ b/test/posix/integration/stat/ftw_tests.cpp @@ -15,6 +15,7 @@ static std::vector items; void ftw_tests() { + INFO("FTW", "ftw_tests"); int res; printf("nftw /mnt/disk/folder1\n"); @@ -119,7 +120,8 @@ int add_filesize(const char *fpath, const struct stat *sb, int flag, struct FTW int add_items(const char *fpath, const struct stat *sb, int flag, struct FTW *ftwbuf) { - printf("add_items: %s\n", fpath); + printf("add_items: %s flag=%i level=%i\n", fpath, flag, ftwbuf->level); + assert(std::find(items.begin(), items.end(), fpath) == items.end() && "No duplicates allowed"); (void) ftwbuf; (void) sb; (void) flag; From fdcf38343bf6adffa3fe5f9b04eb1151480632ad Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 21 Mar 2018 09:34:56 +0100 Subject: [PATCH 092/723] test: add missing symbols to os_mock --- test/lest_util/os_mock.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index 6975a9b7da..08a90f995e 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -110,6 +110,14 @@ void SystemLog::set_flags(uint32_t) {} /// Kernel /// char _binary_apic_boot_bin_end; char _binary_apic_boot_bin_start; +char __plugin_ctors_start; +char __plugin_ctors_end; +char __service_ctors_start; +char __service_ctors_end; +bool __libc_initialized = true; + + + char _ELF_START_; char _ELF_END_; uintptr_t _MULTIBOOT_START_; From eb6d8b5679e50d9fc44f21d9e39be14d6824ad4c Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 21 Mar 2018 11:58:11 +0100 Subject: [PATCH 093/723] linker: add ctors to grouped global ctors --- src/arch/x86_64/linker.ld | 8 ++++---- src/platform/x86_pc/kernel_start.cpp | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/arch/x86_64/linker.ld b/src/arch/x86_64/linker.ld index dba947b00b..21a227c7d7 100644 --- a/src/arch/x86_64/linker.ld +++ b/src/arch/x86_64/linker.ld @@ -71,9 +71,9 @@ SECTIONS .init_array : { PROVIDE_HIDDEN (__init_array_start = .); - */x86_64/lib/lib*.a:*(.init_array*) - */x86_64/platform/lib*.a:*(.init_array*) - */x86_64/drivers/lib*.a:*(.init_array*) + */x86_64/lib/lib*.a:*(.init_array* .ctors*) + */x86_64/platform/lib*.a:*(.init_array* .ctors*) + */x86_64/drivers/lib*.a:*(.init_array* .ctors*) PROVIDE_HIDDEN (__init_array_end = .); } @@ -81,7 +81,7 @@ SECTIONS .plugin_ctors : { PROVIDE_HIDDEN (__plugin_ctors_start = .); - */x86_64/plugins/lib*.a:*(.init_array*) + */x86_64/plugins/lib*.a:*(.init_array* .ctors*) PROVIDE_HIDDEN (__plugin_ctors_end = .); } diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index b426a42671..a74e2a838d 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -52,7 +52,7 @@ thread_local int __tl1__ = 42; uintptr_t __grub_magic = 0xc001; uintptr_t __grub_addr = 0x7001; -static int __global_ctors_ok = 0; +static volatile int __global_ctors_ok = 0; bool __libc_initialized = false; From e4961c763a14b14544de9b110a2ca70fa928419c Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 21 Mar 2018 12:18:41 +0100 Subject: [PATCH 094/723] fiber: fix Wmissing-braces --- src/kernel/fiber.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kernel/fiber.cpp b/src/kernel/fiber.cpp index 918ec48539..069088714d 100644 --- a/src/kernel/fiber.cpp +++ b/src/kernel/fiber.cpp @@ -29,8 +29,8 @@ int Fiber::next_id_{0}; std::atomic Fiber::next_id_{0}; #endif -SMP::Array Fiber::main_ {nullptr}; -SMP::Array Fiber::current_ {nullptr}; +SMP::Array Fiber::main_ = {{nullptr}}; +SMP::Array Fiber::current_ {{nullptr}}; extern "C" { void __fiber_jumpstart(volatile void* th_stack, volatile Fiber* f, volatile void* parent_stack); From f35eb65b8ca52b8d60db30951b6788b04815615d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 21 Mar 2018 13:39:48 +0100 Subject: [PATCH 095/723] musl: Added missing access file --- src/musl/access.cpp | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/musl/access.cpp diff --git a/src/musl/access.cpp b/src/musl/access.cpp new file mode 100644 index 0000000000..563e07115e --- /dev/null +++ b/src/musl/access.cpp @@ -0,0 +1,31 @@ +#include "common.hpp" + +#include + +static long sys_access(const char *pathname, int mode) { + if (UNLIKELY(pathname == nullptr)) + return -EFAULT; + + if (UNLIKELY(pathname[0] == 0)) + return -ENOENT; + + constexpr auto our_flags = R_OK | F_OK; // read only + + try + { + auto ent = fs::VFS::stat_sync(pathname); + if (ent.is_valid()) { + return ((mode & our_flags) == mode) ? 0 : -EROFS; + } + return -ENOENT; + } + catch(const fs::Err_not_found&) + { + return -ENOENT; + } +} + +extern "C" +long syscall_SYS_access(const char *pathname, int mode) { + return strace(sys_access, "access", pathname, mode); +} From ad9580e847bb4b3e0f5e9a755ca644c4b88d8ad9 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 21 Mar 2018 13:49:02 +0100 Subject: [PATCH 096/723] cmake: link with libc again after libos --- cmake/post.service.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 827af1aac3..8b531da90d 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -466,6 +466,7 @@ target_link_libraries(service musl_syscalls libos + libc ${CRTN} ) From 854c562f3c392e9431e56c6fac9bc2adfc1361ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 21 Mar 2018 15:21:11 +0100 Subject: [PATCH 097/723] musl: Fixed getdents (now remembers directory read state) --- api/posix/file_fd.hpp | 5 +- src/posix/file_fd.cpp | 127 ++++++++++++++---------------------------- 2 files changed, 47 insertions(+), 85 deletions(-) diff --git a/api/posix/file_fd.hpp b/api/posix/file_fd.hpp index b2e5ff4495..8765a436eb 100644 --- a/api/posix/file_fd.hpp +++ b/api/posix/file_fd.hpp @@ -24,7 +24,8 @@ class File_FD : public FD { public: explicit File_FD(const int id, const fs::Dirent ent, uint64_t offset = 0) - : FD(id), ent_ {ent}, offset_ {offset} + : FD(id), ent_ {ent}, offset_ {offset}, + dir_pos{0}, dir_{nullptr} {} int read(void*, size_t) override; @@ -39,6 +40,8 @@ class File_FD : public FD { private: fs::Dirent ent_; uint64_t offset_; + size_t dir_pos; + fs::Dirvec_ptr dir_; }; #endif diff --git a/src/posix/file_fd.cpp b/src/posix/file_fd.cpp index 2a6c77a884..8e73222a12 100644 --- a/src/posix/file_fd.cpp +++ b/src/posix/file_fd.cpp @@ -58,79 +58,43 @@ int File_FD::lseek(off_t offset, int whence) return offset_; } - -//struct linux_dirent { -// unsigned long d_ino; /* Inode number */ -// unsigned long d_off; /* Offset to next linux_dirent */ -// unsigned short d_reclen; /* Length of this linux_dirent */ -// char d_name[]; /* Filename (null-terminated) */ -// /* length is actually (d_reclen - 2 - -// offsetof(struct linux_dirent, d_name)) */ -// /* -// char pad; // Zero padding byte -// char d_type; // File type (only since Linux -// // 2.6.4); offset is (d_reclen - 1) -// */ -//} - - - -/* - DT_BLK This is a block device. - - DT_CHR This is a character device. - - DT_DIR This is a directory. - - DT_FIFO This is a named pipe (FIFO). - - DT_LNK This is a symbolic link. - - DT_REG This is a regular file. - - DT_SOCK This is a UNIX domain socket. -*/ -/*struct dirent -{ - ino_t d_ino; - off_t d_off; - unsigned short d_reclen; - unsigned char d_type; - char d_name[256]; -};*/ -struct __dirstream -{ - int fd; - off_t tell; - int buf_pos; - int buf_end; - //volatile int lock[1]; - char buf[2048]; -}; long File_FD::getdents(struct dirent *dirp, unsigned int count) { - static_assert(NAME_MAX+1 == sizeof(dirent::d_name)); - printf("getdents(%p, %u) fd=%i \"%s\" (%s)\n", dirp, count, get_id(), ent_.name().c_str(), ent_.type_string().c_str()); - - auto* dir = (__dirstream*)(((char*)dirp) - sizeof(__dirstream) + sizeof(__dirstream::buf)); - //printf("dirp %p dir %p buf %p\n", dirp, dir, dir->buf); - //assert((void*)dir->buf == (void*)dirp); + static_assert(sizeof(dirent::d_name) > 0); if(not ent_.is_dir()) return -ENOTDIR; + // open (ls) directory if it havent yet + if(dir_ == nullptr) + { + auto list = ent_.ls(); + if(UNLIKELY(list.error)) + return -ENOENT; + + dir_ = std::move(list.entries); + dir_pos = 0; + } + + // we've read every entry + if(dir_pos == dir_->size()) + return 0; + + auto* ptr = reinterpret_cast(dirp); unsigned int written = 0; - auto list = ent_.ls(); - auto* ptr = (char*)dirp; - //ptr -= 4; - for(auto& ent : list) + while(dir_pos != dir_->size()) { - dirp = (dirent*) ptr; - //if(ent.name() == "." or ent.name() == "..") - // continue; + // retreive the next entry from the read directory + const auto& ent = dir_->at(dir_pos); + + // move to next dirent ptr + dirp = reinterpret_cast(ptr); + // inode number dirp->d_ino = ent.block(); + // offset to next dirent (we dont care, i think..) + dirp->d_off = 0; // file type switch(ent.type()) @@ -141,40 +105,35 @@ long File_FD::getdents(struct dirent *dirp, unsigned int count) default: dirp->d_type = DT_REG; } - // offset to next dirent - dirp->d_off = 0; - // filename + // filename (make sure it fits in buffer) auto name = ent.name(); const auto namelen = std::min(sizeof(dirp->d_name)-1, name.length()); - // Length of this dirent - dirp->d_reclen = sizeof(dirent) - sizeof(dirp->d_name) + namelen + 1; + // length of this dirent (struct + name) + const auto len = sizeof(dirent) - sizeof(dirp->d_name) + namelen + 1; - written += dirp->d_reclen; + // if this entry do not fit in the buffer, break + if((written + len) > count) + break; - if(written > count) - return -EINVAL; + // set the length + dirp->d_reclen = len; + + // increase how much we've written + written += dirp->d_reclen; - //std::memcpy(dirp->d_name, name.data(), namelen); - //dirp->d_name[namelen] = '\0'; + // add the filename std::strncpy(dirp->d_name, name.data(), namelen); + // iterate to next ptr += dirp->d_reclen; + dir_pos++; } - //printf("dir %p fd=%u sizeof(buf)=%zu %p\n", dir, dir->fd, sizeof(dir->buf), dir->buf); - dir->buf_end = written; - dir->buf_pos = 0; - - while(dir->buf_pos < dir->buf_end) - { - auto* de = (struct dirent*)(dir->buf + dir->buf_pos); - printf("ino: %zu name: %s type: %u len: %u\n", de->d_ino, de->d_name, de->d_type, de->d_reclen); - if(de->d_reclen == 0) break; - dir->buf_pos += de->d_reclen; - dir->tell = de->d_off; - } + // we haven't written a single entry, which means it didnt fit + if(UNLIKELY(written == 0)) + return -EINVAL; return written; } From eaf0fb6bf4aec7ef2041125ab352781f602d184c Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 26 Mar 2018 10:36:20 +0200 Subject: [PATCH 098/723] src: Fix various 32-bit warnings --- src/drivers/ide.cpp | 3 ++- src/hw/msi.cpp | 2 +- src/kernel/elf.cpp | 14 -------------- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/drivers/ide.cpp b/src/drivers/ide.cpp index d3ecdf18c3..c54ac20f74 100644 --- a/src/drivers/ide.cpp +++ b/src/drivers/ide.cpp @@ -26,6 +26,7 @@ #include #include #include +#include //#define IDE_DEBUG #ifdef IDE_DEBUG @@ -181,7 +182,7 @@ IDE::IDE(hw::PCI_Device& pcidev, selector_t sel) { // 28-bits CHS (MAX_LBA) this->num_blocks = (read_array[61] << 16) | read_array[60]; } - INFO("IDE", "%zu sectors (%zu bytes)", num_blocks, num_blocks * IDE::SECTOR_SIZE); + INFO("IDE", "%" PRIu64 " sectors (%" PRIu64 " bytes)", num_blocks, num_blocks * IDE::SECTOR_SIZE); INFO("IDE", "Initialization complete"); } diff --git a/src/hw/msi.cpp b/src/hw/msi.cpp index 9e956de356..21a294fcbc 100644 --- a/src/hw/msi.cpp +++ b/src/hw/msi.cpp @@ -142,7 +142,7 @@ namespace hw const size_t vector_cnt = (func & MSIX_TBL_SIZE) + 1; if (vector_cnt > 2048) { - printf("table addr: %p pba addr: %p vectors: %lu\n", + printf("table addr: %p pba addr: %p vectors: %zu\n", (void*) table_addr, (void*) pba_addr, vectors()); printf("Unreasonably many MSI-X vectors!"); return; diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index 45e29cfae1..8edef083e2 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -300,20 +300,6 @@ void print_backtrace() void Elf::print_info() { - auto& hdr = elf_header(); - // program headers - auto* phdr = (ElfPhdr*) (ELF_START + hdr.e_phoff); - printf("program headers offs=%#lx at phys %p\n", hdr.e_phoff, phdr); - // section headers - auto* shdr = (ElfShdr*) (ELF_START + hdr.e_shoff); - printf("section headers offs=%#lx at phys %p\n", hdr.e_shoff, shdr); - for (uint16_t i = 0; i < hdr.e_shnum; i++) - { - uintptr_t start = ELF_START + shdr[i].sh_offset; - uintptr_t end = start + shdr[i].sh_size; - printf("sh from %#lx to %#lx\n", start, end); - } - } #include From d67834302d5c5d79a78746a07ae3bf9bb6669905 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 26 Mar 2018 10:36:51 +0200 Subject: [PATCH 099/723] x86: Setup GDT, CPU tables for i686 --- src/platform/x86_pc/platform.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/platform/x86_pc/platform.cpp b/src/platform/x86_pc/platform.cpp index 9f877ba72c..f67e2f117c 100644 --- a/src/platform/x86_pc/platform.cpp +++ b/src/platform/x86_pc/platform.cpp @@ -104,13 +104,20 @@ void __platform_init() kprintf("Platform init done \n"); } -void x86::initialize_cpu_tables_for_cpu(int cpu){ +#ifdef ARCH_i686 +static x86::GDT gdt; +#endif + +void x86::initialize_cpu_tables_for_cpu(int cpu) +{ cpu_tables[cpu].cpuid = cpu; #ifdef ARCH_x86_64 x86::CPU::set_gs(&cpu_tables[cpu]); #else - x86::CPU::set_fs(&cpu_tables[cpu]); + int fs = gdt.create_data(&cpu_tables[cpu], 1); + GDT::reload_gdt(gdt); + GDT::set_fs(fs); #endif } From b9b19e97178fbc8163380b4774c6c7aac5d91d81 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 28 Mar 2018 12:13:39 +0200 Subject: [PATCH 100/723] x86_nano: Musl support, enough to run chainloader --- api/util/alloc_lstack.hpp | 12 +-- cmake/post.service.cmake | 10 +- src/arch/i686/linker.ld | 140 +++++++++++-------------- src/musl/brk.cpp | 12 +-- src/musl/mmap.cpp | 6 +- src/platform/x86_nano/kernel_start.cpp | 44 +++----- src/platform/x86_nano/platform.cpp | 18 +++- src/platform/x86_pc/kernel_start.cpp | 24 ++--- 8 files changed, 116 insertions(+), 150 deletions(-) diff --git a/api/util/alloc_lstack.hpp b/api/util/alloc_lstack.hpp index 3a5db0cd14..dc92642c01 100644 --- a/api/util/alloc_lstack.hpp +++ b/api/util/alloc_lstack.hpp @@ -44,14 +44,14 @@ namespace alloc { * NOTE: The allocator will not search for duplicates on deallocation, * e.g. won't prevent double free, so that has to be done elsewhere. **/ -template +template class Lstack { public: struct Chunk { Chunk* next; - ssize_t size; - Chunk(Chunk* n, ssize_t s) + size_t size; + Chunk(Chunk* n, size_t s) : next(n), size(s) { Expects(s >= Min); @@ -60,7 +60,7 @@ class Lstack { static constexpr int align = Min; - void* allocate(ssize_t size){ + void* allocate(size_t size){ Expects((size & (Min - 1)) == 0); Chunk* next = front_; @@ -98,11 +98,11 @@ class Lstack { push(ptr, size); } - void donate(void* ptr, ssize_t size){ + void donate(void* ptr, size_t size){ push(ptr, size); } - void push(void* ptr, ssize_t size){ + void push(void* ptr, size_t size){ Expects((size & (align - 1)) == 0); auto* old_front = front_; front_ = (Chunk*)ptr; diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 8b531da90d..09446ca56f 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -455,17 +455,9 @@ target_link_libraries(service libunwind libcxx - libplatform - libarch - libos - libc - cxxabi - libpthread - libunwind - libcxx - musl_syscalls libos + libcxx libc ${CRTN} diff --git a/src/arch/i686/linker.ld b/src/arch/i686/linker.ld index 86d10822f4..986bb6131a 100644 --- a/src/arch/i686/linker.ld +++ b/src/arch/i686/linker.ld @@ -23,77 +23,32 @@ SECTIONS PROVIDE ( _ELF_START_ = . + 0x100000); PROVIDE ( _LOAD_START_ = _ELF_START_); /* For convenience w. multiboot */ - . = _ELF_START_; + . = _ELF_START_ + SIZEOF_HEADERS; - .multiboot (_ELF_START_ ): { + .multiboot : { PROVIDE(_MULTIBOOT_START_ = .); *(.multiboot) } - PROVIDE( _TEXT_START_ = . ); - .text ALIGN(0x10) : + .text ALIGN(0x1000): { + PROVIDE( _TEXT_START_ = . ); *(.text) *(.text.*) *(.gnu.linkonce.t*) } PROVIDE( _TEXT_END_ = . ); - .init ALIGN(0x10) : { - _INIT_START_ = .; - *(.init) - _INIT_END_ = .; - } - - .fini ALIGN(0x10) : { - _FINI_START_ = .; - *(.fini) - _FINI_END_ = .; - } - /* Global offset-table. For dynamic linking */ .got ALIGN(0x10) : { *(.got*) } /** - * .ctors, .dtors, .preinit_array, .init_array, .fini_array + * .preinit_array, .init_array, .fini_array * from GNU LD default linker script */ -.ctors : - { - _GCONSTR_START_ = .; - /* gcc uses crtbegin.o to find the start of - the constructors, so we make sure it is - first. Because this is a wildcard, it - doesn't matter if the user does not - actually link against crtbegin.o; the - linker won't look for a file to match a - wildcard. The wildcard also means that it - doesn't matter which directory crtbegin.o - is in. */ - KEEP (*crtbegin.o(.ctors)) - KEEP (*crtbegin?.o(.ctors)) - /* We don't want to include the .ctor section from - the crtend.o file until after the sorted ctors. - The .ctor section from the crtend file contains the - end of ctors marker and it must be last */ - KEEP (*(EXCLUDE_FILE (*crtend.o *crtend?.o ) .ctors)) - KEEP (*(SORT(.ctors.*))) - KEEP (*(.ctors)) - _GCONSTR_END_ = .; - } - - .dtors : - { - KEEP (*crtbegin.o(.dtors)) - KEEP (*crtbegin?.o(.dtors)) - KEEP (*(EXCLUDE_FILE (*crtend.o *crtend?.o ) .dtors)) - KEEP (*(SORT(.dtors.*))) - KEEP (*(.dtors)) - } - .preinit_array : { PROVIDE_HIDDEN (__preinit_array_start = .); @@ -101,15 +56,41 @@ SECTIONS PROVIDE_HIDDEN (__preinit_array_end = .); } +/** + * Global constructors + * Constructors are split into groups allowing the OS to use global ctors + * before the OS itself is initialized, while delaying the calls to service constructors + * until as much of the OS / C++ runtime as possible is ready. + */ + + /* OS / stdlib constructors */ .init_array : { PROVIDE_HIDDEN (__init_array_start = .); + */x86_64/lib/lib*.a:*(.init_array* .ctors*) + */x86_64/platform/lib*.a:*(.init_array* .ctors*) + */x86_64/drivers/lib*.a:*(.init_array* .ctors*) + PROVIDE_HIDDEN (__init_array_end = .); + } + + /* Plugin constructors */ + .plugin_ctors : + { + PROVIDE_HIDDEN (__plugin_ctors_start = .); + */x86_64/plugins/lib*.a:*(.init_array* .ctors*) + PROVIDE_HIDDEN (__plugin_ctors_end = .); + } + + /* All other constructors */ + .service_ctors : + { + PROVIDE_HIDDEN (__service_ctors_start = .); KEEP (*(SORT_BY_INIT_PRIORITY(.init_array.*) SORT_BY_INIT_PRIORITY(.ctors.*))) KEEP (*(.init_array EXCLUDE_FILE (*crtbegin.o *crtbegin?.o *crtend.o *crtend?.o ) .ctors)) - PROVIDE_HIDDEN (__init_array_end = .); + PROVIDE_HIDDEN (__service_ctors_end = .); } - .fini_array : + .fini_array : { PROVIDE_HIDDEN (__fini_array_start = .); KEEP (*(SORT_BY_INIT_PRIORITY(.fini_array.*) SORT_BY_INIT_PRIORITY(.dtors.*))) @@ -117,7 +98,9 @@ SECTIONS PROVIDE_HIDDEN (__fini_array_end = .); } - .config ALIGN(0x8) : { + _EXEC_END_ = .; + _READONLY_START_ = .; + .config ALIGN(0x1000) : { _CONFIG_JSON_START_ = .; KEEP(*(.config)) _CONFIG_JSON_END_ = .; @@ -132,6 +115,28 @@ SECTIONS _RODATA_END_ = .; } + .tdata ALIGN(0x10) : + { + _TDATA_START_ = .; + *(.tdata .tdata.*) + _TDATA_END_ = .; + . = ALIGN(0x10); + } + .tbss : + { + _TBSS_START_ = .; + *(.tbss .tbss.*) + _TBSS_END_ = .; + . = ALIGN(0x10); + } + + .memdisk : + { + _DISK_START_ = .; + *(.diskdata) + _DISK_END_ = .; + } + /* For stack unwinding (exception handling) */ .eh_frame_hdr ALIGN(0x8): { @@ -148,7 +153,7 @@ SECTIONS { *(.gcc_except_table) } - + _READONLY_END_ = .; .data : { _DATA_START_ = .; @@ -158,34 +163,15 @@ SECTIONS _DATA_END_ = .; } - .tdata ALIGN(0x10) : - { - _TDATA_START_ = .; - *(.tdata .tdata.*) - _TDATA_END_ = .; - . = ALIGN(0x10); - } - .tbss : - { - _TBSS_START_ = .; - *(.tbss .tbss.*) - _TBSS_END_ = .; - . = ALIGN(0x10); - } - - .memdisk : - { - _DISK_START_ = .; - *(.diskdata) - _DISK_END_ = .; - } - .elf_symbols : { _ELF_SYM_START_ = .; LONG (0); } - .bss : + /** Optional memory hole between memdisk and bss **/ + . += PRE_BSS_AREA; + + .bss ALIGN(0x1000) : { _BSS_START_ = .; *(.bss .bss.* .gnu.linkonce.b.*) diff --git a/src/musl/brk.cpp b/src/musl/brk.cpp index 10928e656b..d83b8804fb 100644 --- a/src/musl/brk.cpp +++ b/src/musl/brk.cpp @@ -4,11 +4,8 @@ #include #include -__attribute__((weak)) -uintptr_t __brk_max = 0x100000; - +__attribute__((weak)) uintptr_t __brk_max = 0x100000; uintptr_t heap_begin = 0; - uintptr_t brk_end; uintptr_t brk_init = 0; @@ -24,16 +21,14 @@ void _init_heap(uintptr_t free_mem_begin) heap_begin = ((uintptr_t)heap_begin & ~HEAP_ALIGNMENT); brk_end = heap_begin; brk_init = brk_end; - kprintf("* Brk initialized. Begin: 0x%lx, end 0x%lx\n", - heap_begin, brk_end); + kprintf("* Brk initialized. Begin: %p, end %p, MAX %p\n", + (void*) heap_begin, (void*) brk_end, (void*) __brk_max); init_mmap(heap_begin + __brk_max); - } static uintptr_t sys_brk(void* addr) { - if (addr == nullptr or (uintptr_t)addr > heap_begin + __brk_max or (uintptr_t)addr < heap_begin) { @@ -50,7 +45,6 @@ static uintptr_t sys_brk(void* addr) return brk_end; } - extern "C" uintptr_t syscall_SYS_brk(void* addr) { diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 7c75eea0e7..0fa68a4338 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -2,8 +2,9 @@ #include #include #include -#include #include +#include +#include extern uintptr_t heap_begin; extern uintptr_t heap_end; @@ -15,7 +16,8 @@ void init_mmap(uintptr_t addr_begin) { Expects(alloc.empty()); auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); - alloc.donate((void*)aligned_begin, (OS::heap_max() - aligned_begin) & ~(Alloc::align - 1)); + int64_t len = (OS::heap_max() - aligned_begin) & ~int64_t(Alloc::align - 1); + alloc.donate((void*)aligned_begin, len); } extern "C" __attribute__((weak)) diff --git a/src/platform/x86_nano/kernel_start.cpp b/src/platform/x86_nano/kernel_start.cpp index 62e6a11a57..04bf5958cc 100644 --- a/src/platform/x86_nano/kernel_start.cpp +++ b/src/platform/x86_nano/kernel_start.cpp @@ -21,8 +21,6 @@ #include #include -extern void __platform_init(); - extern "C" { void __init_serial1(); void __init_sanity_checks(); @@ -30,16 +28,9 @@ extern "C" { uintptr_t _move_symbols(uintptr_t loc); void _init_bss(); void _init_heap(uintptr_t); - void _init_c_runtime(); void _init_syscalls(); - void __libc_init_array(); - uintptr_t _end; - int __libc_start_main(int (*main)(int,char **,char **), int argc, char **argv); - - } - - +bool __libc_initialized = false; extern "C" void kernel_start(uintptr_t magic, uintptr_t addr) @@ -48,7 +39,8 @@ void kernel_start(uintptr_t magic, uintptr_t addr) __init_serial1(); // Determine where free memory starts - uintptr_t free_mem_begin = reinterpret_cast(&_end); + extern char _end; + uintptr_t free_mem_begin = (uintptr_t) &_end; if (magic == MULTIBOOT_BOOTLOADER_MAGIC) { free_mem_begin = _multiboot_free_begin(addr); @@ -57,36 +49,30 @@ void kernel_start(uintptr_t magic, uintptr_t addr) // Preserve symbols from the ELF binary free_mem_begin += _move_symbols(free_mem_begin); - // Initialize zero-initialized vars - _init_bss(); - - kprintf("Hello musl worldz!\n"); + // Initialize .bss + extern char _BSS_START_, _BSS_END_; + __builtin_memset(&_BSS_START_, 0, &_BSS_END_ - &_BSS_START_); // Initialize heap _init_heap(free_mem_begin); - kprintf("Initialized heap\n"); - - //Initialize stack-unwinder, call global constructors etc. - _init_c_runtime(); - kprintf("C++ exceptions initialized\n"); // Initialize system calls _init_syscalls(); - //Initialize stdout handlers + // Initialize stdout handlers OS::add_stdout(&OS::default_stdout); - kprintf("stdout added\n"); - - // Call global ctors - //__libc_init_array(); - kprintf("Global ctors NOT called - calling printf\n"); - printf("Hello world %s \n", "of musl"); - - __platform_init(); + OS::start(magic, addr); // Start the service Service::start(); __arch_poweroff(); } + +/* +extern "C" int __divdi3() {} +extern "C" int __moddi3() {} +extern "C" unsigned int __udivdi3() {} +extern "C" unsigned int __umoddi3() {} +*/ diff --git a/src/platform/x86_nano/platform.cpp b/src/platform/x86_nano/platform.cpp index ee4338d5bc..972e95519f 100644 --- a/src/platform/x86_nano/platform.cpp +++ b/src/platform/x86_nano/platform.cpp @@ -14,12 +14,21 @@ void __arch_poweroff() __builtin_unreachable(); } -void __platform_init() +static void platform_init() { // setup CPU exception handlers x86::idt_initialize_for_cpu(0); } +void OS::start(uint32_t boot_magic, uint32_t boot_addr) +{ + assert(boot_magic == MULTIBOOT_BOOTLOADER_MAGIC); + OS::multiboot(boot_addr); + assert(OS::memory_end_ != 0); + + platform_init(); +} + // not supported! uint64_t __arch_system_time() noexcept { return 0; @@ -46,11 +55,12 @@ void SMP::global_unlock() noexcept {} int SMP::cpu_id() noexcept { return 0; } int SMP::cpu_count() noexcept { return 1; } -void OS::halt(){ +void OS::halt() { asm("hlt"); } - -// Support for the boot_logger plugin. +// default stdout/logging states __attribute__((weak)) bool os_enable_boot_logging = false; +__attribute__((weak)) +bool os_default_stdout = false; diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 8862ac0919..877fbb9104 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -49,21 +49,18 @@ extern char _FINI_START_; thread_local int __tl1__ = 42; -uintptr_t __grub_magic = 0xc001; -uintptr_t __grub_addr = 0x7001; +uint32_t __grub_magic = 0xc001; +uint32_t __grub_addr = 0x7001; static volatile int __global_ctors_ok = 0; bool __libc_initialized = false; - void _init_bss() { - /// Initialize .bss section extern char _BSS_START_, _BSS_END_; __builtin_memset(&_BSS_START_, 0, &_BSS_END_ - &_BSS_START_); } - __attribute__((constructor)) static void global_ctor_test(){ __global_ctors_ok = 42; @@ -107,22 +104,22 @@ extern "C" uintptr_t __syscall_entry(); extern "C" __attribute__((no_sanitize("all"))) -void kernel_start(uintptr_t magic, uintptr_t addr) +void kernel_start(uint32_t magic, uint32_t addr) { // Initialize default serial port __init_serial1(); __grub_magic = magic; - __grub_addr = addr; + __grub_addr = addr; // TODO: remove extra verbosity after musl port stabilizes kprintf("\n////////////////// IncludeOS kernel start ////////////////// \n"); - kprintf("* Booted with magic 0x%lx, grub @ 0x%lx \n* Init sanity\n", + kprintf("* Booted with magic 0x%x, grub @ 0x%x \n* Init sanity\n", magic, addr); // generate checksums of read-only areas etc. __init_sanity_checks(); - kprintf("* Grub magic: 0x%lx, grub info @ 0x%lx\n", + kprintf("* Grub magic: 0x%x, grub info @ 0x%x\n", __grub_magic, __grub_addr); // Determine where free memory starts @@ -137,9 +134,9 @@ void kernel_start(uintptr_t magic, uintptr_t addr) kprintf("* Moving symbols. \n"); // Preserve symbols from the ELF binary free_mem_begin += _move_symbols(free_mem_begin); - kprintf("* Free mem moved to: 0x%lx \n", free_mem_begin); + kprintf("* Free mem moved to: %p \n", (void*) free_mem_begin); - kprintf("* Grub magic: 0x%lx, grub info @ 0x%lx\n", + kprintf("* Grub magic: 0x%x, grub info @ 0x%x\n", __grub_magic, __grub_addr); kprintf("* Init .bss\n"); @@ -220,7 +217,7 @@ void kernel_start(uintptr_t magic, uintptr_t addr) #if defined(__x86_64__) kprintf("* Initialize syscall MSR\n"); uint64_t star_kernel_cs = 8ull << 32; - uint64_t star_user_cs = 8ull << 48; + uint64_t star_user_cs = 8ull << 48; uint64_t star = star_kernel_cs | star_user_cs; x86::CPU::write_msr(IA32_STAR, star); x86::CPU::write_msr(IA32_LSTAR, (uintptr_t)&__syscall_entry); @@ -229,8 +226,7 @@ void kernel_start(uintptr_t magic, uintptr_t addr) #warning Classical syscall interface missing for 32-bit #endif - //GDB_ENTRY; + // GDB_ENTRY; kprintf("* Starting libc initialization\n"); __libc_start_main(kernel_main, argc, argv.data()); - } From 852d06ec42fd87a5d8ac235af8ed643d75535f45 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 28 Mar 2018 13:47:55 +0200 Subject: [PATCH 101/723] examples: Use user networking for http_client example --- examples/http_client/nacl.txt | 1 + examples/http_client/service.cpp | 15 ++++++++++++--- examples/http_client/vm.json | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/examples/http_client/nacl.txt b/examples/http_client/nacl.txt index 75f41f7e04..caa216ed44 100644 --- a/examples/http_client/nacl.txt +++ b/examples/http_client/nacl.txt @@ -1,5 +1,6 @@ Iface eth0 { index: 0, + config: dhcp-with-fallback, address: 10.0.0.42, netmask: 255.255.255.0, gateway: 10.0.0.1, diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index 647e593a9f..a4bd712c00 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -37,12 +37,11 @@ static SSL_CTX* init_ssl_context() #include #include -void Service::start() +static void begin_http() { - auto& inet = net::Super_stack::get(0); - using namespace http; + auto& inet = net::Super_stack::get(0); static Basic_client basic{inet.tcp()}; const std::string url{"http://www.google.com"}; @@ -78,3 +77,13 @@ void Service::start() } }); } + +void Service::start() +{ + auto& inet = net::Super_stack::get(0); + + inet.on_config( + [] (auto) { + begin_http(); + }); +} diff --git a/examples/http_client/vm.json b/examples/http_client/vm.json index d0dae980d0..877347b2f7 100644 --- a/examples/http_client/vm.json +++ b/examples/http_client/vm.json @@ -1,5 +1,5 @@ { "net" : [ - {"device" : "virtio", "mac" : "c0:01:0a:00:00:2a"} + {"device" : "virtio", "mac" : "c0:01:0a:00:00:2a", "backend": "user"} ] } From 4de7ccd6c77bc10078e325ad0621a9f05946ceb1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 13:49:08 +0200 Subject: [PATCH 102/723] crt: Add libgcc overflow checked functions --- src/crt/c_abi.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index 83c323f55a..99d164b734 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -17,6 +17,7 @@ #include #include +#include #include #define HEAP_ALIGNMENT 63 @@ -37,3 +38,83 @@ uint32_t _move_symbols(void* sym_loc) return elfsym_size; } + +void* __memcpy_chk(void* dest, const void* src, size_t len, size_t destlen) +{ + assert (len <= destlen); + return memcpy(dest, src, len); +} +void* __memset_chk(void* dest, int c, size_t len, size_t destlen) +{ + assert (len <= destlen); + return memset(dest, c, len); +} +char* __strcat_chk(char* dest, const char* src, size_t destlen) +{ + size_t len = strlen(dest) + strlen(src) + 1; + assert (len <= destlen); + return strcat(dest, src); +} + +__attribute__((format(printf, 2, 3))) +int __printf_chk (int flag, const char *format, ...) +{ + (void) flag; + va_list ap; + va_start (ap, format); + int done = vfprintf (stdout, format, ap); + va_end (ap); + return done; +} +int __fprintf_chk(FILE* fp, int flag, const char* format, ...) +{ + (void) flag; + va_list arg; + va_start (arg, format); + int done = vfprintf(fp, format, arg); + va_end (arg); + return done; +} +int __vfprintf_chk(FILE* fp, int flag, const char *format, va_list ap) +{ + (void) flag; + int done; + done = vfprintf (fp, format, ap); + return done; +} +int __vsprintf_chk(char* s, int flag, size_t slen, const char* format, va_list args) +{ + (void) flag; + int res = vsnprintf(s, slen, format, args); + assert ((size_t) res < slen); + return res; +} +__attribute__((format(printf, 4, 5))) +int __sprintf_chk(char* s, int flags, size_t slen, const char *format, ...) +{ + va_list arg; + int done; + va_start (arg, format); + done = __vsprintf_chk(s, flags, slen, format, arg); + va_end (arg); + return done; +} + +int __isoc99_scanf (const char *format, ...) +{ + va_list arg; + va_start (arg, format); + int done = vfscanf(stdin, format, arg); + va_end (arg); + return done; +} +__attribute__((format(scanf, 2, 3))) +int __isoc99_sscanf (const char *s, const char *format, ...) +{ + va_list arg; + int done; + va_start (arg, format); + done = vsscanf(s, format, arg); + va_end (arg); + return done; +} From e50d4e95b10be670c43e44135239feb683cc59fd Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 14:01:01 +0200 Subject: [PATCH 103/723] musl: read/write correct error on unknown FD --- src/musl/read.cpp | 3 +-- src/musl/write.cpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/musl/read.cpp b/src/musl/read.cpp index 6cfc19d833..ff05e14963 100644 --- a/src/musl/read.cpp +++ b/src/musl/read.cpp @@ -7,8 +7,7 @@ static ssize_t sys_read(int fd, void* buf, size_t len) { return fildes.read(buf, len); } catch(const FD_not_found&) { - errno = EBADF; - return -1; + return -EBADF; } } diff --git a/src/musl/write.cpp b/src/musl/write.cpp index bc4601d48d..84a3ed495f 100644 --- a/src/musl/write.cpp +++ b/src/musl/write.cpp @@ -16,8 +16,7 @@ static long sys_write(int fd, char* str, size_t len) { return fildes.write(str, len); } catch(const FD_not_found&) { - errno = EBADF; - return -1; + return -EBADF; } } From 99e5b3781c0ea181183a6034870fce0c61bdc188 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 14:06:59 +0200 Subject: [PATCH 104/723] musl: Use long in read syscall --- src/musl/read.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/musl/read.cpp b/src/musl/read.cpp index ff05e14963..a409b0c464 100644 --- a/src/musl/read.cpp +++ b/src/musl/read.cpp @@ -1,7 +1,7 @@ #include "common.hpp" #include -static ssize_t sys_read(int fd, void* buf, size_t len) { +static long sys_read(int fd, void* buf, size_t len) { try { auto& fildes = FD_map::_get(fd); return fildes.read(buf, len); @@ -12,6 +12,6 @@ static ssize_t sys_read(int fd, void* buf, size_t len) { } extern "C" -ssize_t syscall_SYS_read(int fd, void *buf, size_t nbyte) { +long syscall_SYS_read(int fd, void *buf, size_t nbyte) { return strace(sys_read, "read", fd, buf, nbyte); } From c7a9851aa17c617d5aa2f71bc526b2204c17c30d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 14:10:06 +0200 Subject: [PATCH 105/723] posix: Assert on FD map close --- api/posix/fd_map.hpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/api/posix/fd_map.hpp b/api/posix/fd_map.hpp index cb15df0b50..a3ad0e7e2a 100644 --- a/api/posix/fd_map.hpp +++ b/api/posix/fd_map.hpp @@ -55,9 +55,6 @@ class FD_map { FD& get(const id_t id) const; - int close(const id_t id) - { return (map_.erase(id)) ? 0 : -1; } - template static auto&& _open(Args&&... args) { return instance().open(std::forward(args)...); } @@ -65,10 +62,16 @@ class FD_map { static auto&& _get(const id_t id) { return instance().get(id); } - static int _close(id_t id) - { return instance().close(id); } + static void close(id_t id) + { instance().internal_close(id); } private: + void internal_close(const id_t id) + { + auto erased = map_.erase(id); + assert(erased > 0); + } + std::map> map_; FD_map() {} }; From c42dcce4b7620e066f059c10d830092c7b9b6a95 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 14:14:40 +0200 Subject: [PATCH 106/723] musl: Implement close syscall --- src/musl/close.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/musl/close.cpp b/src/musl/close.cpp index 58690a9e33..3da8f186e0 100644 --- a/src/musl/close.cpp +++ b/src/musl/close.cpp @@ -1,10 +1,18 @@ #include "common.hpp" +#include -static int sys_close(int fd) { - if (!fd) return -1; - return 0; - -}; +static long sys_close(int fd) +{ + try { + auto& fildes = FD_map::_get(fd); + int res = fildes.close(); + FD_map::close(fd); + return res; + } + catch(const FD_not_found&) { + return -EBADF; + } +} extern "C" int syscall_SYS_close(int fd) { From c69b0b90fbe8ad7f7654756e0fb49d28de2a7ea3 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 14:15:08 +0200 Subject: [PATCH 107/723] musl: Add getuid, rt_sigaction, socketcall stubs --- src/musl/CMakeLists.txt | 5 +++-- src/musl/getuid.cpp | 10 ++++++++++ src/musl/rt_sigaction.cpp | 19 +++++++++++++++++++ src/musl/socketcall.cpp | 17 +++++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/musl/getuid.cpp create mode 100644 src/musl/rt_sigaction.cpp create mode 100644 src/musl/socketcall.cpp diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index 5e4893a476..af5c90f208 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -6,8 +6,9 @@ set(MUSL_OBJECTS getrlimit.cpp sched_yield.cpp set_robust_list.cpp nanosleep.cpp open.cpp creat.cpp clock_gettime.cpp gettimeofday.cpp poll.cpp exit.cpp close.cpp set_tid_address.cpp - pipe.cpp read.cpp readv.cpp getpid.cpp mknod.cpp sync.cpp msync.cpp - mincore.cpp syscall_n.cpp sigmask.cpp gettid.cpp tkill.cpp + pipe.cpp read.cpp readv.cpp getpid.cpp getuid.cpp mknod.cpp sync.cpp + msync.cpp mincore.cpp syscall_n.cpp sigmask.cpp gettid.cpp tkill.cpp + socketcall.cpp rt_sigaction.cpp stat.cpp fstat.cpp fstatat.cpp chmod.cpp cwd.cpp diff --git a/src/musl/getuid.cpp b/src/musl/getuid.cpp new file mode 100644 index 0000000000..a7ef0bb52a --- /dev/null +++ b/src/musl/getuid.cpp @@ -0,0 +1,10 @@ +#include "common.hpp" + +static long sys_getuid() { + return 0; +} + +extern "C" +long syscall_SYS_getuid() { + return strace(sys_getuid, "getuid"); +} diff --git a/src/musl/rt_sigaction.cpp b/src/musl/rt_sigaction.cpp new file mode 100644 index 0000000000..fb149e237e --- /dev/null +++ b/src/musl/rt_sigaction.cpp @@ -0,0 +1,19 @@ +#include "common.hpp" + +static int sys_sigaction(int signum, + const struct sigaction* act, + const struct sigaction* oldact) +{ + (void) signum; + (void) act; + (void) oldact; + return -ENOSYS; +} + +extern "C" +int syscall_SYS_rt_sigaction(int signum, + const struct sigaction* act, + const struct sigaction* oldact) +{ + return strace(sys_sigaction, "rt_sigaction", signum, act, oldact); +} diff --git a/src/musl/socketcall.cpp b/src/musl/socketcall.cpp new file mode 100644 index 0000000000..c00e3b210c --- /dev/null +++ b/src/musl/socketcall.cpp @@ -0,0 +1,17 @@ +#include + +extern "C" +long socketcall_socket() +{ + return -ENOSYS; +} +extern "C" +long socketcall_connect() +{ + return -ENOSYS; +} +extern "C" +long socketcall_shutdown() +{ + return -ENOSYS; +} From e918bf8dc9129eb3d83808b816fa66be02aa112d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 14:17:51 +0200 Subject: [PATCH 108/723] i686: Remove unused sentinel table for i686 --- src/arch/i686/arch_start.asm | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/arch/i686/arch_start.asm b/src/arch/i686/arch_start.asm index fbf25303bf..da80700c8b 100644 --- a/src/arch/i686/arch_start.asm +++ b/src/arch/i686/arch_start.asm @@ -43,11 +43,3 @@ __arch_start: mov esp, ebp pop ebp ret - -sentinel_table: - dd sentinel_table ;; 0x0 - dd 0 ;; 0x8 - dd 0 ;; 0x10 - dd 0 ;; 0x18 - dd 0 ;; 0x20 - dd 0x123456789ABCDEF From dbd74d663ebd3afc92d34a50d1970b379f876676 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 16:29:00 +0200 Subject: [PATCH 109/723] posix: Add fstat() to RNG devices --- api/posix/rng_fd.hpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/api/posix/rng_fd.hpp b/api/posix/rng_fd.hpp index 5bc6fc6743..4eba4f8518 100644 --- a/api/posix/rng_fd.hpp +++ b/api/posix/rng_fd.hpp @@ -21,6 +21,7 @@ #include #include "fd.hpp" +#include /** * @brief Simple stateless file descriptor @@ -43,6 +44,20 @@ class RNG_fd : public FD { return bytes; } + long fstat(struct stat* buf) override + { + buf->st_dev = 0x6; + buf->st_ino = 10; + buf->st_nlink = 1; + buf->st_size = 0; + buf->st_atime = 0; + buf->st_ctime = 0; + buf->st_mtime = 0; + buf->st_blocks = 0; + buf->st_blksize = 4096; + return 0; + } + int close() override { return 0; } From 713e9b16418c077ca5002444714ac33edb017c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 3 Apr 2018 16:30:10 +0200 Subject: [PATCH 110/723] fs: Pretend FAT root directory can be stat-ed --- src/fs/fat_async.cpp | 3 +-- src/fs/fat_sync.cpp | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/fs/fat_async.cpp b/src/fs/fat_async.cpp index 7e299b7b4d..31c75aa187 100644 --- a/src/fs/fat_async.cpp +++ b/src/fs/fat_async.cpp @@ -226,9 +226,8 @@ namespace fs { // manual lookup if (UNLIKELY(path->empty())) { - // root doesn't have any stat anyways // Note: could use ATTR_VOLUME_ID in FAT - func({ error_t::E_NOENT, "Cannot stat root" }, Dirent(this, INVALID_ENTITY, path->to_string())); + func(no_error, Dirent(this, DIR, "/", 0)); return; } diff --git a/src/fs/fat_sync.cpp b/src/fs/fat_sync.cpp index 702d1c80e5..55b738fac8 100644 --- a/src/fs/fat_sync.cpp +++ b/src/fs/fat_sync.cpp @@ -149,8 +149,7 @@ namespace fs Dirent FAT::stat(Path path, const Dirent* const start) const { if (UNLIKELY(path.empty())) { - // root doesn't have any stat anyways (except ATTR_VOLUME_ID in FAT) - return Dirent(this, INVALID_ENTITY); + return Dirent(this, Enttype::DIR, "/", 0); } FS_PRINT("stat_sync: %s\n", path.back().c_str()); From 33c75425859d3cd1947372bf08812aa1204c69c6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 16:30:16 +0200 Subject: [PATCH 111/723] openssl: Temp fixes for musl --- examples/TLS_server/CMakeLists.txt | 1 + src/net/openssl/init.cpp | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/TLS_server/CMakeLists.txt b/examples/TLS_server/CMakeLists.txt index f25fd0d4bf..8cede98e12 100644 --- a/examples/TLS_server/CMakeLists.txt +++ b/examples/TLS_server/CMakeLists.txt @@ -38,6 +38,7 @@ endif() set(PLUGINS autoconf # configuration from config.json + vfs # ...others ) diff --git a/src/net/openssl/init.cpp b/src/net/openssl/init.cpp index cd6e6a4a81..1f2414a4c6 100644 --- a/src/net/openssl/init.cpp +++ b/src/net/openssl/init.cpp @@ -65,9 +65,8 @@ namespace openssl init_once = true; SSL_library_init(); OpenSSL_add_all_algorithms(); - SSL_load_error_strings(); + //SSL_load_error_strings(); ERR_load_BIO_strings(); - ERR_load_crypto_strings(); } } } From 138dfe4f887745854c27851ff5b541b6d69a1269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 3 Apr 2018 16:33:14 +0200 Subject: [PATCH 112/723] fs: Fixed stack reference bug in VFS --- api/fs/vfs.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/fs/vfs.hpp b/api/fs/vfs.hpp index cb22b0f5bb..17d5caa670 100644 --- a/api/fs/vfs.hpp +++ b/api/fs/vfs.hpp @@ -347,7 +347,7 @@ namespace fs { disk, remote, insert_dirent_delg::make_packed( - [local, callback, desc](error_t err, auto&& dirent_ref) + [local, callback, desc](error_t err, auto& dirent_ref) { VFS::mount(local, dirent_ref, desc); callback(err); @@ -439,7 +439,7 @@ namespace fs { auto res_it = (dirent_map().emplace(Dirent_mountpoint{disk_id, path.to_string()}, dir)); if (res_it.second) { - auto saved_dir = res_it.first->second; + auto& saved_dir = res_it.first->second; fn(err, saved_dir); } else { fn(err, invalid_dirent()); From 023337efe162d760fdec201b0e64be8a54edd5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 3 Apr 2018 16:34:03 +0200 Subject: [PATCH 113/723] plugin: Support mounting memdisk in VFS with help of config --- src/plugins/vfs.cpp | 85 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/src/plugins/vfs.cpp b/src/plugins/vfs.cpp index bb11cd242c..f085dcb017 100644 --- a/src/plugins/vfs.cpp +++ b/src/plugins/vfs.cpp @@ -21,6 +21,89 @@ #include #include +#ifndef RAPIDJSON_HAS_STDSTRING + #define RAPIDJSON_HAS_STDSTRING 1 +#endif + +#include +#include + +#include +static hw::Block_device& get_block_device(const std::string& str) +{ + if(str == "memdisk") + { + auto& disk = fs::memdisk(); + return disk.dev(); + } + + throw std::runtime_error{"Unsupported disk type"}; +} + +#include +static void parse_config() +{ + const auto& json = ::Config::get(); + + // Nothing to do with empty config + if(json.empty()) + return; + + using namespace rapidjson; + Document doc; + doc.Parse(json.data()); + + Expects(doc.IsObject() && "Malformed config"); + + // No vfs member, ignore + if(not doc.HasMember("vfs")) + return; + + const auto& cfg = doc["vfs"]; + + Expects(cfg.IsArray() && "Member vfs is not an array"); + + auto mounts = cfg.GetArray(); + if(mounts.Size() == 0) + { + INFO("VFS Mounter", "NOTE: No VFS entries - nothing to do"); + return; + } + + INFO("VFS Mounter", "Found %u VFS entries", mounts.Size()); + + for(auto& val : mounts) + { + Expects(val.HasMember("disk") && "Missing disk"); + Expects(val.HasMember("root") && "Missing root"); + Expects(val.HasMember("mount") && "Missing mount"); + Expects(val.HasMember("description") && "Missing description"); + + auto& dev = get_block_device(val["disk"].GetString()); + auto disk = fs::VFS::insert_disk(dev); + + // TODO: maybe move this code to get_block_device + if (not disk->fs_ready()) + { + disk->init_fs([](fs::error_t err, auto&) { + Expects(not err && "Error mounting disk"); + }); + } + + Expects(disk->fs_ready() && "Filesystem not ready (async disk?)"); + + fs::VFS::mount( + fs::Path{val["mount"].GetString()}, + dev.id(), + fs::Path{val["root"].GetString()}, + val["description"].GetString(), + [](auto err) { + Expects(not err && "Error occured while mounting"); + }); + } + +} + #include // Mount RNG functionality to system paths static void mount_rng() @@ -38,6 +121,8 @@ static void vfs_mount() { // RNG mount_rng(); + + parse_config(); } #include From d5de4eb6de4c2345d54eba3ecd90f8e8bb0b2421 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 3 Apr 2018 16:34:55 +0200 Subject: [PATCH 114/723] test: Replace VFS helper function with plugin in posix stat --- test/posix/integration/stat/CMakeLists.txt | 11 ++++++++-- test/posix/integration/stat/config.json | 10 ++++++++++ test/posix/integration/stat/test_stat_ftw.cpp | 20 ------------------- 3 files changed, 19 insertions(+), 22 deletions(-) create mode 100644 test/posix/integration/stat/config.json diff --git a/test/posix/integration/stat/CMakeLists.txt b/test/posix/integration/stat/CMakeLists.txt index 32e34a0e48..1dabfadc4c 100644 --- a/test/posix/integration/stat/CMakeLists.txt +++ b/test/posix/integration/stat/CMakeLists.txt @@ -10,13 +10,20 @@ project(test_posix_stat) set(SERVICE_NAME "Stat test") set(BINARY "test_posix_stat") -set(MAX_MEM 128) + set(SOURCES test_stat_ftw.cpp ftw_tests.cpp stat_tests.cpp ) -#set(LOCAL_INCLUDES ".") + +set(DRIVERS + boot_logger +) + +set(PLUGINS + vfs +) # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/posix/integration/stat/config.json b/test/posix/integration/stat/config.json new file mode 100644 index 0000000000..e7a81540b0 --- /dev/null +++ b/test/posix/integration/stat/config.json @@ -0,0 +1,10 @@ +{ + "vfs" : [ + { + "disk" : "memdisk", + "root" : "/", + "mount" : "/mnt/disk/", + "description" : "test root" + } + ] +} diff --git a/test/posix/integration/stat/test_stat_ftw.cpp b/test/posix/integration/stat/test_stat_ftw.cpp index 024240cc8d..d037162e2b 100644 --- a/test/posix/integration/stat/test_stat_ftw.cpp +++ b/test/posix/integration/stat/test_stat_ftw.cpp @@ -16,7 +16,6 @@ // limitations under the License. #include -#include #include #include #include @@ -24,29 +23,10 @@ int ftw_tests(); int stat_tests(); -fs::Disk& memdisk() { - fs::Disk& disk = fs::memdisk(); - - if (not disk.fs_ready()) { - printf("%s\n", disk.name().c_str()); - disk.init_fs([](fs::error_t err, auto&) { - if (err) { - printf("ERROR MOUNTING DISK\n"); - exit(127); - } - }); - } - return disk; -} - int main() { INFO("POSIX stat", "Running tests for POSIX stat"); - // mount a disk with contents for testing - auto root = memdisk().fs().stat("/"); - fs::mount("/mnt/disk", root, "test root"); - fs::print_tree(); stat_tests(); From 95ff292f77c1d2d0d03ab083b6003d6ef3ce3917 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 23:28:41 +0200 Subject: [PATCH 115/723] x86: Refactor APIC interrupt paths --- src/platform/x86_pc/apic.cpp | 78 ++++++++++++++++++------------ src/platform/x86_pc/apic.hpp | 4 +- src/platform/x86_pc/apic_iface.hpp | 9 ++-- src/platform/x86_pc/x2apic.hpp | 31 +++++++----- src/platform/x86_pc/xapic.hpp | 22 +++++++-- 5 files changed, 91 insertions(+), 53 deletions(-) diff --git a/src/platform/x86_pc/apic.cpp b/src/platform/x86_pc/apic.cpp index dd7b9e249e..99b773eca1 100644 --- a/src/platform/x86_pc/apic.cpp +++ b/src/platform/x86_pc/apic.cpp @@ -34,6 +34,18 @@ namespace x86 IApic& APIC::get() noexcept { return *current_apic; } + static void apic_handle_intr(int vector) + { + Events::get().trigger_event(vector - IRQ_BASE); +#ifdef ENABLE_DYNAMIC_EOI + assert(current_eoi_mechanism != nullptr); + current_eoi_mechanism(); +#endif + } + // shortcut that avoids virtual call + static void x2apic_send_eoi() { + CPU::write_msr(x86::x2apic::BASE_MSR + x2APIC_EOI, 0); + } } extern "C" { @@ -41,33 +53,37 @@ extern "C" { void (*current_eoi_mechanism)() = nullptr; void (*real_eoi_mechanism)() = nullptr; void (*current_intr_handler)() = nullptr; - // shortcut that avoids virtual call - void x2apic_send_eoi() { - x86::CPU::write_msr(x86::x2apic::BASE_MSR + x2APIC_EOI, 0); - } // the various interrupt handler flavors - void xapic_intr_handler() + static void xapic_intr_handler() { - uint8_t vector = x86::APIC::get_isr(); - //assert(vector >= IRQ_BASE && vector < 160); - Events::get().trigger_event(vector - IRQ_BASE); -#ifdef ENABLE_DYNAMIC_EOI - assert(current_eoi_mechanism != nullptr); - current_eoi_mechanism(); -#else - lapic_send_eoi(); + for (int i = 1; i < 5; i++) { + uint32_t reg = x86::xapic::get_isr_at(i); + if (reg) { + const int vector = __builtin_ffs(reg) - 1; + // handle interrupt vector + x86::apic_handle_intr(32 * i + vector); + break; + } + } +#ifndef ENABLE_DYNAMIC_EOI + // fixed location when APIC is not moved + *(volatile uint32_t*) 0xfee000B0 = 0; #endif } - void x2apic_intr_handler() + static void x2apic_intr_handler() { - uint8_t vector = x86::x2apic::static_get_isr(); - //assert(vector >= IRQ_BASE && vector < 160); - Events::get().trigger_event(vector - IRQ_BASE); -#ifdef ENABLE_DYNAMIC_EOI - assert(current_eoi_mechanism != nullptr); - current_eoi_mechanism(); -#else - x2apic_send_eoi(); + for (int i = 1; i < 5; i++) { + uint32_t reg = x86::x2apic::get_isr_at(i); + if (reg) { + const int vector = __builtin_ffs(reg) - 1; + // handle interrupt vector + x86::apic_handle_intr(32 * i + vector); + break; + } + } +#ifndef ENABLE_DYNAMIC_EOI + // EOI fast-path + x86::x2apic_send_eoi(); #endif } } @@ -83,6 +99,9 @@ namespace x86 PIC::init(); PIC::disable(); + // initialize I/O APICs + IOAPIC::init(ACPI::get_ioapics()); + if (CPUID::has_feature(CPUID::Feature::X2APIC)) { current_apic = &x2apic::get(); real_eoi_mechanism = x2apic_send_eoi; @@ -96,20 +115,17 @@ namespace x86 current_intr_handler = xapic_intr_handler; } - if (current_eoi_mechanism == nullptr) - current_eoi_mechanism = real_eoi_mechanism; - - // enable xAPIC/x2APIC on this cpu - current_apic->enable(); - - // initialize I/O APICs - IOAPIC::init(ACPI::get_ioapics()); - #ifdef ENABLE_KVM_PV_EOI // use KVMs paravirt EOI if supported if (CPUID::kvm_feature(KVM_FEATURE_PV_EOI)) kvm_pv_eoi_init(); #endif + + if (current_eoi_mechanism == nullptr) + current_eoi_mechanism = real_eoi_mechanism; + + // enable xAPIC/x2APIC on this cpu + current_apic->enable(); } void APIC::enable_irq(uint8_t irq) diff --git a/src/platform/x86_pc/apic.hpp b/src/platform/x86_pc/apic.hpp index 6702be4bd8..78f08991ad 100644 --- a/src/platform/x86_pc/apic.hpp +++ b/src/platform/x86_pc/apic.hpp @@ -34,10 +34,10 @@ namespace x86 static void enable_irq (uint8_t irq); static void disable_irq(uint8_t irq); - static uint8_t get_isr() noexcept { + static int get_isr() noexcept { return get().get_isr(); } - static uint8_t get_irr() noexcept { + static int get_irr() noexcept { return get().get_irr(); } static void eoi() noexcept { diff --git a/src/platform/x86_pc/apic_iface.hpp b/src/platform/x86_pc/apic_iface.hpp index 5086e4200e..150595b360 100644 --- a/src/platform/x86_pc/apic_iface.hpp +++ b/src/platform/x86_pc/apic_iface.hpp @@ -19,6 +19,9 @@ #ifndef X86_APIC_IFACE_HPP #define X86_APIC_IFACE_HPP +#include +#include + namespace x86 { class IApic { @@ -36,9 +39,9 @@ namespace x86 { virtual void enable() noexcept = 0; virtual void smp_enable() noexcept = 0; - virtual void eoi() noexcept = 0; - virtual uint8_t get_isr() noexcept = 0; - virtual uint8_t get_irr() noexcept = 0; + virtual void eoi() noexcept = 0; + virtual int get_isr() noexcept = 0; + virtual int get_irr() noexcept = 0; virtual void ap_init (int id) noexcept = 0; virtual void ap_start(int id, uint32_t vec) noexcept = 0; diff --git a/src/platform/x86_pc/x2apic.hpp b/src/platform/x86_pc/x2apic.hpp index 99353e5cfe..40e65a785d 100644 --- a/src/platform/x86_pc/x2apic.hpp +++ b/src/platform/x86_pc/x2apic.hpp @@ -49,11 +49,6 @@ #define IA32_APIC_BASE_MSR 0x1B -extern "C" { - extern void (*current_eoi_mechanism)(); - extern void x2apic_send_eoi(); -} - namespace x86 { struct x2apic : public IApic @@ -135,34 +130,46 @@ namespace x86 { enable(); } - static uint8_t static_get_isr() noexcept + static int static_get_isr() noexcept { for (int i = 5; i >= 1; i--) { uint32_t reg = CPU::read_msr(BASE_MSR + x2APIC_ISR + i); if (reg) return 32 * i + __builtin_ffs(reg) - 1; } - return 159; + return -1; } void eoi() noexcept override { write(x2APIC_EOI, 0); } - uint8_t get_isr() noexcept override + int get_isr() noexcept override { for (int i = 5; i >= 1; i--) { uint32_t reg = read(x2APIC_ISR + i); if (reg) return 32 * i + __builtin_ffs(reg) - 1; } - return 159; + return -1; } - uint8_t get_irr() noexcept override + int get_irr() noexcept override { - for (int i = 5; i >= 1; i--) { + for (int i = 5; i >= 0; i--) { uint32_t reg = read(x2APIC_IRR + i); if (reg) return 32 * i + __builtin_ffs(reg) - 1; } - return 159; + return -1; + } + static uint32_t get_isr_at(int index) noexcept + { + return CPU::read_msr(BASE_MSR + x2APIC_ISR + index); + } + static std::array get_isr_array() noexcept + { + std::array isr_array; + for (int i = 1; i < 6; i++) { + isr_array[i] = get_isr_at(i); + } + return isr_array; } void ap_init(int id) noexcept override diff --git a/src/platform/x86_pc/xapic.hpp b/src/platform/x86_pc/xapic.hpp index 8716f8e197..7eaa3a931b 100644 --- a/src/platform/x86_pc/xapic.hpp +++ b/src/platform/x86_pc/xapic.hpp @@ -133,21 +133,33 @@ namespace x86 { { write(xAPIC_EOI, 0); } - uint8_t get_isr() noexcept override + int get_isr() noexcept override { for (int i = 5; i >= 1; i--) { uint32_t reg = read(xAPIC_ISR + 0x10 * i); if (reg) return 32 * i + __builtin_ffs(reg) - 1; } - return 159; + return -1; } - uint8_t get_irr() noexcept override + int get_irr() noexcept override { - for (int i = 5; i >= 1; i--) { + for (int i = 5; i >= 0; i--) { uint32_t reg = read(xAPIC_IRR + 0x10 * i); if (reg) return 32 * i + __builtin_ffs(reg) - 1; } - return 159; + return -1; + } + static uint32_t get_isr_at(int index) noexcept + { + return *(volatile uint32_t*) uintptr_t(0xfee00000 + xAPIC_ISR + 0x10 * index); + } + static std::array get_isr_array() noexcept + { + std::array isr_array; + for (int i = 1; i < 6; i++) { + isr_array[i] = get_isr_at(i); + } + return isr_array; } void ap_init(int id) noexcept override From 814bdffa4dab4f6a50d6cc1c1349bd25f764de04 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 23:29:13 +0200 Subject: [PATCH 116/723] kvm: Make PV-EOI SMP-aware --- src/platform/kvm/pv_eoi.cpp | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/platform/kvm/pv_eoi.cpp b/src/platform/kvm/pv_eoi.cpp index 1ac79d4f62..8d5090616a 100644 --- a/src/platform/kvm/pv_eoi.cpp +++ b/src/platform/kvm/pv_eoi.cpp @@ -1,7 +1,14 @@ #include #include +#include "../x86_pc/apic.hpp" #include "../x86_pc/cpu.hpp" #include +#include + +// current selected EOI method +extern void (*current_eoi_mechanism)(); +extern void (*real_eoi_mechanism)(); +extern void (*current_intr_handler)(); // *** manual *** // http://choon.net/forum/read.php?21,1123399 @@ -13,35 +20,30 @@ #define KVM_PV_EOI_MASK (0x1 << KVM_PV_EOI_BIT) #define KVM_PV_EOI_ENABLED KVM_PV_EOI_MASK -// current selected EOI method -extern void (*current_eoi_mechanism)(); -extern void (*real_eoi_mechanism)(); -extern void (*current_intr_handler)(); - -__attribute__ ((aligned(4))) -static volatile unsigned long kvm_exitless_eoi = 0; +struct alignas(SMP_ALIGN) pv_eoi +{ + uint32_t eoi_word = 0; +}; +static SMP_ARRAY exitless_eoi; extern "C" void kvm_pv_eoi() { - if (kvm_exitless_eoi) { - kprintf("avoidable eoi\n"); - } uint8_t reg; - asm("btr %2, %0; setc %1" : "+m"(kvm_exitless_eoi), "=rm"(reg) : "r"(0)); + asm("btr %2, %0; setc %1" : "+m"(PER_CPU(exitless_eoi).eoi_word), "=rm"(reg) : "r"(0)); if (reg) { kprintf("avoidable eoi\n"); return; } else { - //kprintf("unavoidable eoi\n"); + //kprintf("APIC IRR: %d\n", x86::APIC::get_irr()); } // fallback to normal x2APIC EOI real_eoi_mechanism(); } void kvm_pv_eoi_init() { - uint64_t addr = (uint64_t) &kvm_exitless_eoi; + uint64_t addr = (uint64_t) &PER_CPU(exitless_eoi).eoi_word; addr |= KVM_MSR_ENABLED; x86::CPU::write_msr(MSR_KVM_PV_EOI_EN, addr); // verify that the feature was enabled From 36ae02f3303c404db234c1112920a3f9b5101851 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Apr 2018 23:30:06 +0200 Subject: [PATCH 117/723] vmrunner: Make CPU model a regular string --- vmrunner/vm.schema.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/vmrunner/vm.schema.json b/vmrunner/vm.schema.json index 705317567d..1bde0fc7d3 100644 --- a/vmrunner/vm.schema.json +++ b/vmrunner/vm.schema.json @@ -7,12 +7,10 @@ "description" : "The virtual CPU", "type" : "object", "properties": { - "model" : { - "enum" : ["host", "pentium", "qemu64", "kvm64"] - }, + "model" : { "type" : "string" }, "features" : { "type" : "array", - "items" : { "enum" : ["xsave", "avx", "aes"] } + "items" : { "type" : "string" } } } } From 1a47c8c7e0232c168269b0edfb0a5a0b33394081 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 4 Apr 2018 10:48:19 +0200 Subject: [PATCH 118/723] musl: Impl readv, improved read overall --- api/posix/fd.hpp | 6 ++++-- api/posix/file_fd.hpp | 3 ++- api/posix/rng_fd.hpp | 2 +- src/musl/close.cpp | 2 +- src/musl/read.cpp | 6 +++++- src/musl/readv.cpp | 18 ++++++++++++------ src/posix/file_fd.cpp | 29 ++++++++++++++++++++++++++++- 7 files changed, 53 insertions(+), 13 deletions(-) diff --git a/api/posix/fd.hpp b/api/posix/fd.hpp index 6cab026e4e..b6546f76ee 100644 --- a/api/posix/fd.hpp +++ b/api/posix/fd.hpp @@ -23,6 +23,7 @@ #include #include #include +#include /** * @brief File descriptor @@ -38,7 +39,8 @@ class FD { {} /** FILES **/ - virtual int read(void*, size_t) { return -1; } + virtual ssize_t read(void*, size_t) { return -EBADF; } + virtual ssize_t readv(const struct iovec*, int) { return -EBADF; } virtual int write(const void*, size_t) { return -1; } virtual int close() = 0; virtual int fcntl(int, va_list); @@ -72,7 +74,7 @@ class FD { virtual int lseek(off_t, int) { return -1; } // linux specific - virtual long getdents([[maybe_unused]]struct dirent *dirp, [[maybe_unused]]unsigned int count) { return -1; } + virtual long getdents(struct dirent*, unsigned int) { return -1; } id_t get_id() const noexcept { return id_; } diff --git a/api/posix/file_fd.hpp b/api/posix/file_fd.hpp index 8765a436eb..9903e65581 100644 --- a/api/posix/file_fd.hpp +++ b/api/posix/file_fd.hpp @@ -28,7 +28,8 @@ class File_FD : public FD { dir_pos{0}, dir_{nullptr} {} - int read(void*, size_t) override; + ssize_t read(void*, size_t) override; + ssize_t readv(const struct iovec*, int iovcnt) override; int write(const void*, size_t) override; int close() override; int lseek(off_t, int) override; diff --git a/api/posix/rng_fd.hpp b/api/posix/rng_fd.hpp index 4eba4f8518..369b7f2459 100644 --- a/api/posix/rng_fd.hpp +++ b/api/posix/rng_fd.hpp @@ -32,7 +32,7 @@ class RNG_fd : public FD { RNG_fd(int fd) : FD{fd} {} - int read(void* output, size_t bytes) override + ssize_t read(void* output, size_t bytes) override { rng_extract(output, bytes); return bytes; diff --git a/src/musl/close.cpp b/src/musl/close.cpp index 3da8f186e0..174d14f508 100644 --- a/src/musl/close.cpp +++ b/src/musl/close.cpp @@ -15,6 +15,6 @@ static long sys_close(int fd) } extern "C" -int syscall_SYS_close(int fd) { +long syscall_SYS_close(int fd) { return strace(sys_close, "close", fd); } diff --git a/src/musl/read.cpp b/src/musl/read.cpp index a409b0c464..4c70943284 100644 --- a/src/musl/read.cpp +++ b/src/musl/read.cpp @@ -1,7 +1,11 @@ #include "common.hpp" #include -static long sys_read(int fd, void* buf, size_t len) { +static long sys_read(int fd, void* buf, size_t len) +{ + if(UNLIKELY(buf == nullptr)) + return -EFAULT; + try { auto& fildes = FD_map::_get(fd); return fildes.read(buf, len); diff --git a/src/musl/readv.cpp b/src/musl/readv.cpp index f5782f7f27..9220232546 100644 --- a/src/musl/readv.cpp +++ b/src/musl/readv.cpp @@ -1,14 +1,20 @@ -#include "stub.hpp" -#include -#include +#include "common.hpp" -static ssize_t sys_readv(int /*fd*/, const struct iovec*, int /*iovcnt*/) +#include + +static ssize_t sys_readv(int fd, const struct iovec* iov, int iovcnt) { - return -ENOSYS; + try { + auto& fildes = FD_map::_get(fd); + return fildes.readv(iov, iovcnt); + } + catch(const FD_not_found&) { + return -EBADF; + } } extern "C" ssize_t syscall_SYS_readv(int fd, const struct iovec *iov, int iovcnt) { - return stubtrace(sys_readv, "readv", fd, iov, iovcnt); + return strace(sys_readv, "readv", fd, iov, iovcnt); } diff --git a/src/posix/file_fd.cpp b/src/posix/file_fd.cpp index 8e73222a12..f35c4fdd19 100644 --- a/src/posix/file_fd.cpp +++ b/src/posix/file_fd.cpp @@ -18,14 +18,41 @@ #include #include #include +#include + +ssize_t File_FD::read(void* p, size_t n) +{ + Expects(p != nullptr); + + if(UNLIKELY(ent_.is_dir())) + return -EISDIR; + + if(UNLIKELY(n == 0)) + return 0; -int File_FD::read(void* p, size_t n) { auto buf = ent_.read(offset_, n); memcpy(p, buf.data(), std::min(n, buf.size())); offset_ += buf.size(); return buf.size(); } +ssize_t File_FD::readv(const struct iovec* iov, int iovcnt) +{ + if(UNLIKELY(iovcnt <= 0 and iovcnt > IOV_MAX)) + return -EINVAL; + + ssize_t total = 0; + // TODO: return EINVAL if total overflow ssize_t + for(int i = 0; i < iovcnt; i++) + { + auto& vec = iov[i]; + if(vec.iov_len > 0) + total += this->read(vec.iov_base, vec.iov_len); + } + + return total; +} + int File_FD::write(const void*, size_t) { return -1; } From 1074c0ad834d8fe150c1582f5944d284901e950a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 4 Apr 2018 10:55:50 +0200 Subject: [PATCH 119/723] musl: stubbed socket sendmsg --- src/musl/socketcall.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/musl/socketcall.cpp b/src/musl/socketcall.cpp index c00e3b210c..3826190ba9 100644 --- a/src/musl/socketcall.cpp +++ b/src/musl/socketcall.cpp @@ -1,3 +1,5 @@ +#include "common.hpp" +#include "stub.hpp" #include extern "C" @@ -15,3 +17,15 @@ long socketcall_shutdown() { return -ENOSYS; } + +static long sock_sendmsg(int /*sockfd*/, const struct msghdr */*msg*/, int /*flags*/) +{ + return -ENOSYS; +} + +extern "C" +long socketcall_sendmsg(int sockfd, const struct msghdr *msg, int flags) +{ + return stubtrace(sock_sendmsg, "sendmsg", sockfd, msg, flags); +} + From 9722b1edc82d07c21a5f2fe188db6ef8606deafb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 4 Apr 2018 10:56:38 +0200 Subject: [PATCH 120/723] test: posix/conf now using vfs plugin --- test/posix/integration/conf/CMakeLists.txt | 4 ++++ test/posix/integration/conf/config.json | 10 ++++++++++ test/posix/integration/conf/service.cpp | 21 --------------------- 3 files changed, 14 insertions(+), 21 deletions(-) create mode 100644 test/posix/integration/conf/config.json diff --git a/test/posix/integration/conf/CMakeLists.txt b/test/posix/integration/conf/CMakeLists.txt index bcf2b4928c..b0d8451c47 100644 --- a/test/posix/integration/conf/CMakeLists.txt +++ b/test/posix/integration/conf/CMakeLists.txt @@ -35,6 +35,10 @@ set(DRIVERS # ... Others from IncludeOS/src/drivers ) +set(PLUGINS + vfs + ) + # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/posix/integration/conf/config.json b/test/posix/integration/conf/config.json new file mode 100644 index 0000000000..59bdf27afd --- /dev/null +++ b/test/posix/integration/conf/config.json @@ -0,0 +1,10 @@ +{ + "vfs" : [ + { + "disk" : "memdisk", + "root" : "/", + "mount" : "/etc", + "description" : "test fs" + } + ] +} diff --git a/test/posix/integration/conf/service.cpp b/test/posix/integration/conf/service.cpp index 12b7281710..abc4a9cfc1 100644 --- a/test/posix/integration/conf/service.cpp +++ b/test/posix/integration/conf/service.cpp @@ -22,28 +22,11 @@ #include #include #include -#include -#include extern "C" void test_sysconf(); extern "C" void test_pathconf(); extern "C" void test_pwd(); -fs::Disk& memdisk() { - fs::Disk& disk = fs::memdisk(); - - if (not disk.fs_ready()) { - disk.init_fs([](fs::error_t err, auto&) { - if (err) { - printf("ERROR MOUNTING DISK\n"); - printf("%s\n", err.reason().c_str()); - exit(127); - } - }); - } - return disk; -} - int main(int, char **) { struct utsname name; @@ -56,10 +39,6 @@ int main(int, char **) { printf("%s %s\n", name.sysname, name.version); test_sysconf(); - - // mount a disk with contents for testing - auto root = memdisk().fs().stat("/"); - fs::mount("/etc", root, "test fs"); test_pathconf(); // test environment variables From 9038a7137b53621c826924a5b06e1f3bfba97bf6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 4 Apr 2018 11:10:45 +0200 Subject: [PATCH 121/723] net: Use OS::page_size() in buffer store --- src/net/buffer_store.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net/buffer_store.cpp b/src/net/buffer_store.cpp index f488d8cad3..9a855a8309 100644 --- a/src/net/buffer_store.cpp +++ b/src/net/buffer_store.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include #include @@ -56,7 +56,7 @@ namespace net { assert(bufsize != 0); const size_t DATA_SIZE = poolsize_; - this->pool_ = (uint8_t*) aligned_alloc(PAGE_SIZE, DATA_SIZE); + this->pool_ = (uint8_t*) aligned_alloc(OS::page_size(), DATA_SIZE); assert(this->pool_); available_.reserve(num); From 5e871d1f1ae6d45535a545e1c3b11c0e7e8ee7e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 5 Apr 2018 10:29:51 +0200 Subject: [PATCH 122/723] musl: Implemented lseek, updated file_fd test --- api/posix/fd.hpp | 9 +++---- api/posix/file_fd.hpp | 2 +- src/musl/lseek.cpp | 22 ++++++++++------- src/posix/file_fd.cpp | 5 ++-- test/posix/integration/file_fd/CMakeLists.txt | 7 ++++-- test/posix/integration/file_fd/config.json | 10 ++++++++ .../integration/file_fd/test_file_fd.cpp | 24 +------------------ 7 files changed, 38 insertions(+), 41 deletions(-) create mode 100644 test/posix/integration/file_fd/config.json diff --git a/api/posix/fd.hpp b/api/posix/fd.hpp index b6546f76ee..c07661324b 100644 --- a/api/posix/fd.hpp +++ b/api/posix/fd.hpp @@ -25,6 +25,7 @@ #include #include +#define DEFAULT_ERR EPERM /** * @brief File descriptor * @details @@ -39,9 +40,9 @@ class FD { {} /** FILES **/ - virtual ssize_t read(void*, size_t) { return -EBADF; } - virtual ssize_t readv(const struct iovec*, int) { return -EBADF; } - virtual int write(const void*, size_t) { return -1; } + virtual ssize_t read(void*, size_t) { return -DEFAULT_ERR; } + virtual ssize_t readv(const struct iovec*, int) { return -DEFAULT_ERR; } + virtual int write(const void*, size_t) { return -DEFAULT_ERR; } virtual int close() = 0; virtual int fcntl(int, va_list); virtual int ioctl(int, void*); @@ -71,7 +72,7 @@ class FD { virtual int mkdirat(const char *, mode_t) { return -1; } virtual int mkfifoat(const char *, mode_t) { return -1; } virtual int mknodat(const char *, mode_t, dev_t) { return -1; } - virtual int lseek(off_t, int) { return -1; } + virtual off_t lseek(off_t, int) { return -DEFAULT_ERR; } // linux specific virtual long getdents(struct dirent*, unsigned int) { return -1; } diff --git a/api/posix/file_fd.hpp b/api/posix/file_fd.hpp index 9903e65581..6703276c99 100644 --- a/api/posix/file_fd.hpp +++ b/api/posix/file_fd.hpp @@ -32,7 +32,7 @@ class File_FD : public FD { ssize_t readv(const struct iovec*, int iovcnt) override; int write(const void*, size_t) override; int close() override; - int lseek(off_t, int) override; + off_t lseek(off_t, int) override; long getdents(struct dirent *dirp, unsigned int count) override; diff --git a/src/musl/lseek.cpp b/src/musl/lseek.cpp index d6b46eda66..e100c0f1a1 100644 --- a/src/musl/lseek.cpp +++ b/src/musl/lseek.cpp @@ -1,29 +1,35 @@ +#include "common.hpp" #include #include #include #include "stub.hpp" +#include -off_t sys_lseek(int /*fd*/, off_t /*offset*/, int /*whence*/) { - errno = ENOSYS; - return -1; +static off_t sys_lseek(int fd, off_t offset, int whence) { + try { + auto& fildes = FD_map::_get(fd); + return fildes.lseek(offset, whence); + } + catch(const FD_not_found&) { + return -EBADF; + } } -off_t sys__llseek(unsigned int /*fd*/, unsigned long /*offset_high*/, +static off_t sys__llseek(unsigned int /*fd*/, unsigned long /*offset_high*/, unsigned long /*offset_low*/, loff_t* /*result*/, unsigned int /*whence*/) { - errno = ENOSYS; - return -1; + return -ENOSYS; } extern "C" off_t syscall_SYS_lseek(int fd, off_t offset, int whence) { - return stubtrace(sys_lseek, "lseek", fd, offset, whence); + return strace(sys_lseek, "lseek", fd, offset, whence); } extern "C" -int syscall_SYS__llseek(unsigned int fd, unsigned long offset_high, +off_t syscall_SYS__llseek(unsigned int fd, unsigned long offset_high, unsigned long offset_low, loff_t *result, unsigned int whence) { return stubtrace(sys__llseek, "_llseek", fd, offset_high, offset_low, diff --git a/src/posix/file_fd.cpp b/src/posix/file_fd.cpp index f35c4fdd19..c67b1b376d 100644 --- a/src/posix/file_fd.cpp +++ b/src/posix/file_fd.cpp @@ -61,12 +61,11 @@ int File_FD::close() { return 0; } -int File_FD::lseek(off_t offset, int whence) +off_t File_FD::lseek(off_t offset, int whence) { if ((whence != SEEK_SET) && (whence != SEEK_CUR) && (whence != SEEK_END)) { //PRINT("lseek(%lu, %d) == %d\n", offset, whence, -1); - errno = EINVAL; - return -1; + return -EINVAL; } off_t calculated_offset = offset_; if (whence == SEEK_SET) { diff --git a/test/posix/integration/file_fd/CMakeLists.txt b/test/posix/integration/file_fd/CMakeLists.txt index 1f6b50d98b..56f7477754 100644 --- a/test/posix/integration/file_fd/CMakeLists.txt +++ b/test/posix/integration/file_fd/CMakeLists.txt @@ -10,11 +10,14 @@ project(test_file_fd) set(SERVICE_NAME "File_fd test") set(BINARY "test_file_fd") -set(MAX_MEM 128) + set(SOURCES test_file_fd.cpp ) -#set(LOCAL_INCLUDES ".") + +set(PLUGINS + vfs + ) set_property(SOURCE test_file_fd.cpp APPEND_STRING PROPERTY COMPILE_FLAGS " -Wno-unused-function ") diff --git a/test/posix/integration/file_fd/config.json b/test/posix/integration/file_fd/config.json new file mode 100644 index 0000000000..cfc6d0fb93 --- /dev/null +++ b/test/posix/integration/file_fd/config.json @@ -0,0 +1,10 @@ +{ + "vfs" : [ + { + "disk" : "memdisk", + "root" : "/", + "mount" : "/mnt/disk", + "description" : "test root" + } + ] +} diff --git a/test/posix/integration/file_fd/test_file_fd.cpp b/test/posix/integration/file_fd/test_file_fd.cpp index 7748a63306..77d2a08854 100644 --- a/test/posix/integration/file_fd/test_file_fd.cpp +++ b/test/posix/integration/file_fd/test_file_fd.cpp @@ -17,32 +17,14 @@ #include #include -#include -#include #include #include #include #include #include - #include -fs::Disk_ptr& memdisk() { - static auto disk = fs::shared_memdisk(); - - if (not disk->fs_ready()) { - disk->init_fs([](fs::error_t err, auto&) { - if (err) { - printf("ERROR MOUNTING DISK\n"); - printf("%s\n", err.reason().c_str()); - exit(127); - } - }); - } - return disk; -} - const lest::test specification[] = { { @@ -294,7 +276,7 @@ const lest::test specification[] = size_t wanted_count {7}; size_t count = fread(buf, sizeof(char), wanted_count, f); EXPECT(count == wanted_count); - printf("fread result: %d\n", count); + printf("fread result: %zu\n", count); int cmp = strcmp(buf, "content"); EXPECT(cmp == 0); int res = fclose(f); @@ -321,10 +303,6 @@ int main() { INFO("POSIX file_fd", "Running tests for POSIX file_fd"); - // mount a disk with contents for testing - auto root = memdisk()->fs().stat("/"); - fs::mount("/mnt/disk", root, "test root"); - auto failed = lest::run(specification, {"-p"}); Expects(not failed); From ab8a2c91245800aa481134bb0003ecf608a20a37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 5 Apr 2018 11:34:16 +0200 Subject: [PATCH 123/723] test: removed deprecated ucontext test --- .../posix/integration/ucontext/CMakeLists.txt | 30 ------ test/posix/integration/ucontext/README.md | 1 - test/posix/integration/ucontext/service.cpp | 93 ------------------- test/posix/integration/ucontext/test.py | 12 --- test/posix/integration/ucontext/vm.json | 1 - 5 files changed, 137 deletions(-) delete mode 100644 test/posix/integration/ucontext/CMakeLists.txt delete mode 100644 test/posix/integration/ucontext/README.md delete mode 100644 test/posix/integration/ucontext/service.cpp delete mode 100755 test/posix/integration/ucontext/test.py delete mode 100644 test/posix/integration/ucontext/vm.json diff --git a/test/posix/integration/ucontext/CMakeLists.txt b/test/posix/integration/ucontext/CMakeLists.txt deleted file mode 100644 index 2a78eb61f4..0000000000 --- a/test/posix/integration/ucontext/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -cmake_minimum_required(VERSION 2.8.9) - -# IncludeOS install location -if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) - set(ENV{INCLUDEOS_PREFIX} /usr/local) -endif() - -include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) - -project (test_ucontext) - -# Human-readable name of your service -set(SERVICE_NAME "POSIX ucontext Test Service") - -# Name of your service binary -set(BINARY "test_ucontext") - -# Maximum memory can be hard-coded into the binary -set(MAX_MEM 128) - -# Source files to be linked with OS library parts to form bootable image -set(SOURCES - service.cpp - ) - -# To add your own include paths: -#set(LOCAL_INCLUDES ".") - -# include service build script -include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/posix/integration/ucontext/README.md b/test/posix/integration/ucontext/README.md deleted file mode 100644 index 52e66590c1..0000000000 --- a/test/posix/integration/ucontext/README.md +++ /dev/null @@ -1 +0,0 @@ -# POSIX ucontext_t integration test diff --git a/test/posix/integration/ucontext/service.cpp b/test/posix/integration/ucontext/service.cpp deleted file mode 100644 index 57fe145f5a..0000000000 --- a/test/posix/integration/ucontext/service.cpp +++ /dev/null @@ -1,93 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include - -#define STACK_ALIGN __attribute__((aligned(16))) -static ucontext_t foo_context; -static ucontext_t bar_context; -static ucontext_t main_context; -static unsigned int bar_context_var = -1; -static unsigned int foo_context_var = -1; - -void bar(int argc) -{ - printf("Successfuly jumped into 'bar' context\n"); - - Expects(argc == 0); - - bar_context_var = 0xDEADBEEF; - - printf("'bar' context returning successfully\n"); -} - -void foo(int argc, int arg1, int arg2) -{ - printf("Successfuly jumped into 'foo' context\n"); - - Expects(argc == 2); - Expects(arg1 == -2414); - Expects(arg2 == ~0); - - foo_context_var = 0xFEEDDEAD; - - - volatile bool swapped = false; - - if(!swapped) { - swapped = true; - Expects(swapcontext(&foo_context, &bar_context) != -1); - } - - Expects(bar_context_var == 0xDEADBEEF); - - printf("'foo' context returning succesfully\n"); -} - -void Service::start() -{ - INFO("ucontext_t", "Testing POSIX ucontext_t"); - - uint8_t foo_stack[1024] STACK_ALIGN; - uint8_t bar_stack[1024] STACK_ALIGN; - - Expects(getcontext(&foo_context) != -1); - foo_context.uc_link = &main_context; - foo_context.uc_stack.ss_sp = foo_stack + sizeof(foo_stack)-32; - foo_context.uc_stack.ss_size = sizeof(bar_stack)-32; - makecontext(&foo_context, (void(*)())foo, 2, -2414, ~0); - - Expects(getcontext(&bar_context) != -1); - bar_context.uc_link = &foo_context; - bar_context.uc_stack.ss_sp = bar_stack + sizeof(bar_stack); - bar_context.uc_stack.ss_size = sizeof(bar_stack); - makecontext(&bar_context, (void(*)())bar, 0); - - volatile bool swapped = false; - getcontext(&main_context); - - if(!swapped) { - swapped = true; - Expects(swapcontext(&main_context, &foo_context) != -1); - } - - Expects(foo_context_var == 0xFEEDDEAD); - - INFO("ucontext ","SUCCESS"); -} diff --git a/test/posix/integration/ucontext/test.py b/test/posix/integration/ucontext/test.py deleted file mode 100755 index c10e8306d8..0000000000 --- a/test/posix/integration/ucontext/test.py +++ /dev/null @@ -1,12 +0,0 @@ -#! /usr/bin/env python -import sys -import os - -includeos_src = os.environ.get('INCLUDEOS_SRC', - os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) -sys.path.insert(0,includeos_src) - -from vmrunner import vmrunner - -vm = vmrunner.vms[0]; -vm.cmake().boot(20).clean() diff --git a/test/posix/integration/ucontext/vm.json b/test/posix/integration/ucontext/vm.json deleted file mode 100644 index 6cff6ff865..0000000000 --- a/test/posix/integration/ucontext/vm.json +++ /dev/null @@ -1 +0,0 @@ -{"image" : "test_ucontext.img" } From c83d7df27a3dbe4c1c77bed6b9e26728c9727bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 5 Apr 2018 11:35:06 +0200 Subject: [PATCH 124/723] musl: WIP on socketcalls --- src/musl/socketcall.cpp | 128 ++++++++++++++++++++++--- test/posix/integration/udp/service.cpp | 1 + 2 files changed, 118 insertions(+), 11 deletions(-) diff --git a/src/musl/socketcall.cpp b/src/musl/socketcall.cpp index 3826190ba9..2c5b35564e 100644 --- a/src/musl/socketcall.cpp +++ b/src/musl/socketcall.cpp @@ -1,31 +1,137 @@ #include "common.hpp" -#include "stub.hpp" #include +#include -extern "C" -long socketcall_socket() +#include + +static long sock_socket(int domain, int type, int protocol) { - return -ENOSYS; + // disallow strange domains, like ALG + if (UNLIKELY(domain < 0 || domain > AF_INET6)) + return -EAFNOSUPPORT; + // disallow RAW etc + if (UNLIKELY(type < 0 || type > SOCK_DGRAM)) + return -EINVAL; + // we are purposefully ignoring the protocol argument + if (UNLIKELY(protocol < 0)) + return -EPROTONOSUPPORT; + + /*return [](const int type)->int{ + switch(type) + { + case SOCK_STREAM: + return FD_map::_open().get_id(); + case SOCK_DGRAM: + return FD_map::_open().get_id(); + default: + errno = EINVAL; + return -1; + } + }(type);*/ + return -EINVAL; +} + +static long sock_connect(int sockfd, const struct sockaddr *addr, + socklen_t addrlen) +{ + try { + auto& fildes = FD_map::_get(sockfd); + return fildes.connect(addr, addrlen); + } + catch(const FD_not_found&) { + return -EBADF; + } +} + +static long sock_bind(int sockfd, const struct sockaddr *addr, + socklen_t addrlen) +{ + try { + auto& fildes = FD_map::_get(sockfd); + return fildes.connect(addr, addrlen); + } + catch(const FD_not_found&) { + return -EBADF; + } } + +static long sock_sendmsg(int sockfd, const struct msghdr *msg, int flags) +{ + try { + auto& fildes = FD_map::_get(sockfd); + return fildes.sendmsg(msg, flags); + } + catch(const FD_not_found&) { + return -EBADF; + } +} + +static ssize_t sock_sendto(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen) +{ + try { + auto& fildes = FD_map::_get(sockfd); + return fildes.sendto(buf, len, flags, dest_addr, addrlen); + } + catch(const FD_not_found&) { + return -EBADF; + } +} + +static ssize_t sock_recvfrom(int sockfd, void *buf, size_t len, int flags, + struct sockaddr *src_addr, socklen_t *addrlen) +{ + try { + auto& fildes = FD_map::_get(sockfd); + return fildes.recvfrom(buf, len, flags, src_addr, addrlen); + } + catch(const FD_not_found&) { + return -EBADF; + } +} + extern "C" -long socketcall_connect() +long socketcall_socket(int domain, int type, int protocol) { - return -ENOSYS; + return strace(sock_socket, "socket", domain, type, protocol); } + extern "C" -long socketcall_shutdown() +long socketcall_connect(int sockfd, const struct sockaddr *addr, + socklen_t addrlen) { - return -ENOSYS; + return strace(sock_connect, "connect", sockfd, addr, addrlen); } -static long sock_sendmsg(int /*sockfd*/, const struct msghdr */*msg*/, int /*flags*/) +extern "C" +long socketcall_bind(int sockfd, const struct sockaddr *addr, + socklen_t addrlen) { - return -ENOSYS; + return strace(sock_bind, "bind", sockfd, addr, addrlen); } extern "C" long socketcall_sendmsg(int sockfd, const struct msghdr *msg, int flags) { - return stubtrace(sock_sendmsg, "sendmsg", sockfd, msg, flags); + return strace(sock_sendmsg, "sendmsg", sockfd, msg, flags); +} + +extern "C" +ssize_t socketcall_sendto(int sockfd, const void *buf, size_t len, int flags, + const struct sockaddr *dest_addr, socklen_t addrlen) +{ + return strace(sock_sendto, "sendto", sockfd, buf, len, flags, dest_addr, addrlen); } +extern "C" +ssize_t socketcall_recvfrom(int sockfd, void *buf, size_t len, int flags, + struct sockaddr *src_addr, socklen_t *addrlen) +{ + return strace(sock_recvfrom, "recvfrom", sockfd, buf, len, flags, src_addr, addrlen); +} + +extern "C" +long socketcall_shutdown() +{ + return -ENOSYS; +} diff --git a/test/posix/integration/udp/service.cpp b/test/posix/integration/udp/service.cpp index 8c41f417ab..e5eb2e25e1 100644 --- a/test/posix/integration/udp/service.cpp +++ b/test/posix/integration/udp/service.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include From f37e1898001c6c5c67107eb746881c5c9474f195 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 5 Apr 2018 12:11:06 +0200 Subject: [PATCH 125/723] musl: Implement madvise, enough for musl internals --- src/musl/madvise.cpp | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/musl/madvise.cpp b/src/musl/madvise.cpp index e29bfbaa86..adfad77511 100644 --- a/src/musl/madvise.cpp +++ b/src/musl/madvise.cpp @@ -1,7 +1,30 @@ #include "common.hpp" +#include + +static long sys_madvise(void* addr, size_t length, int advice) +{ + switch (advice) + { + case MADV_NORMAL: + case MADV_RANDOM: + case MADV_SEQUENTIAL: + case MADV_WILLNEED: + return 0; // no-op + case MADV_DONTNEED: + // addr:len is unused, but the application hasn't freed the range + return 0; + case MADV_REMOVE: + case MADV_FREE: + printf("madvise with free/remove called on %p:%zu\n", addr, length); + // TODO: free range + return 0; + default: + printf("madvise with %d called on %p:%zu\n", advice, addr, length); + return -EINVAL; + } +} extern "C" -long syscall_SYS_madvise() { - STUB("madvise"); - return 0; +long syscall_SYS_madvise(void* addr, size_t length, int advice) { + return strace(sys_madvise, "madvise", addr, length, advice); } From bd5c77f5650d973596c28d3dcd443fa755b3f101 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 5 Apr 2018 12:18:20 +0200 Subject: [PATCH 126/723] openssl: Initialize without adding all algos --- src/net/openssl/init.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/net/openssl/init.cpp b/src/net/openssl/init.cpp index 8f0e1c5c53..45b819efdb 100644 --- a/src/net/openssl/init.cpp +++ b/src/net/openssl/init.cpp @@ -65,9 +65,9 @@ namespace openssl { INFO("OpenSSL", "Initializing (%s)", OPENSSL_VERSION_TEXT); init_once = true; - SSL_library_init(); - OpenSSL_add_all_algorithms(); - //SSL_load_error_strings(); + SSL_library_init(); // OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS + SSL_load_error_strings(); + ERR_load_crypto_strings(); ERR_load_BIO_strings(); } } From b96923002db45a7ed9bdf9c42e6a8b9e20004f22 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 5 Apr 2018 12:31:04 +0200 Subject: [PATCH 127/723] openssl: Use IncludeOS RNG directly --- src/net/openssl/init.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/net/openssl/init.cpp b/src/net/openssl/init.cpp index 45b819efdb..58ece7007e 100644 --- a/src/net/openssl/init.cpp +++ b/src/net/openssl/init.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -40,7 +41,7 @@ namespace openssl { void setup_rng() { - RAND_METHOD ios_rand { + static RAND_METHOD ios_rand { ios_rand_seed, ios_rand_bytes, ios_rand_cleanup, @@ -65,6 +66,7 @@ namespace openssl { INFO("OpenSSL", "Initializing (%s)", OPENSSL_VERSION_TEXT); init_once = true; + setup_rng(); SSL_library_init(); // OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS SSL_load_error_strings(); ERR_load_crypto_strings(); From 55d403014cd1645d6838b2876a6b616da3ecf159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 5 Apr 2018 14:16:10 +0200 Subject: [PATCH 128/723] fd: Removed need for try/catch by returning fd pointer --- api/posix/fd_map.hpp | 27 +++++++++++++------- src/musl/close.cpp | 14 +++++------ src/musl/common.hpp | 4 +-- src/musl/fcntl.cpp | 13 +++++----- src/musl/fstat.cpp | 13 ++++------ src/musl/fstatat.cpp | 15 +++++------ src/musl/getdents.cpp | 13 +++++----- src/musl/ioctl.cpp | 12 +++------ src/musl/lseek.cpp | 14 +++++------ src/musl/read.cpp | 11 +++------ src/musl/readv.cpp | 11 +++------ src/musl/socketcall.cpp | 55 +++++++++++++++-------------------------- src/musl/write.cpp | 11 +++------ 13 files changed, 92 insertions(+), 121 deletions(-) diff --git a/api/posix/fd_map.hpp b/api/posix/fd_map.hpp index a3ad0e7e2a..ab1a33b825 100644 --- a/api/posix/fd_map.hpp +++ b/api/posix/fd_map.hpp @@ -43,6 +43,7 @@ class FD_not_found : public FD_map_error { class FD_map { public: using id_t = FD::id_t; + using FD_ptr = std::unique_ptr; static FD_map& instance() { @@ -53,13 +54,13 @@ class FD_map { template T& open(Args&&... args); - FD& get(const id_t id) const; + FD* get(const id_t id) const noexcept; template - static auto&& _open(Args&&... args) + static T& _open(Args&&... args) { return instance().open(std::forward(args)...); } - static auto&& _get(const id_t id) + static FD* _get(const id_t id) noexcept { return instance().get(id); } static void close(id_t id) @@ -84,17 +85,25 @@ T& FD_map::open(Args&&... args) static id_t counter = 5; const auto id = counter++; + auto* fd = new T(id, std::forward(args)...); - map_.emplace(std::piecewise_construct, std::forward_as_tuple(id), std::forward_as_tuple(fd)); + + auto res = map_.emplace(std::piecewise_construct, + std::forward_as_tuple(id), + std::forward_as_tuple(fd)); + + if(UNLIKELY(res.second == false)) + throw FD_map_error{"Unable to open FD (map emplace failed)"}; + return *fd; } -inline FD& FD_map::get(const id_t id) const +inline FD* FD_map::get(const id_t id) const noexcept { - auto it = map_.find(id); - if(it != map_.end()) - return *it->second; - throw FD_not_found{id}; + if(auto it = map_.find(id); it != map_.end()) + return it->second.get(); + + return nullptr; } #endif diff --git a/src/musl/close.cpp b/src/musl/close.cpp index 174d14f508..b1a2637736 100644 --- a/src/musl/close.cpp +++ b/src/musl/close.cpp @@ -3,15 +3,15 @@ static long sys_close(int fd) { - try { - auto& fildes = FD_map::_get(fd); - int res = fildes.close(); + if(auto* fildes = FD_map::_get(fd); fildes) + { + long res = fildes->close(); + if(res != 0) + return res; FD_map::close(fd); - return res; - } - catch(const FD_not_found&) { - return -EBADF; + return 0; } + return -EBADF; } extern "C" diff --git a/src/musl/common.hpp b/src/musl/common.hpp index 416dbdf650..532873c0ad 100644 --- a/src/musl/common.hpp +++ b/src/musl/common.hpp @@ -25,7 +25,7 @@ inline constexpr auto& pr_param(std::ostream& out, L lhs, Args&&... rest){ // avoid writing nullptr to std out if constexpr(std::is_pointer_v) { if(lhs != nullptr) out << lhs << ", "; - else out << "nullptr, "; + else out << "NULL, "; } else { out << lhs << ", "; @@ -37,7 +37,7 @@ inline constexpr auto& pr_param(std::ostream& out, L lhs, Args&&... rest){ // avoid writing nullptr to std out if constexpr(std::is_pointer_v) { if(lhs != nullptr) out << lhs; - else out << "nullptr"; + else out << "NULL"; } else { out << lhs; diff --git a/src/musl/fcntl.cpp b/src/musl/fcntl.cpp index 0bf1c59b43..9f0c3e05d9 100644 --- a/src/musl/fcntl.cpp +++ b/src/musl/fcntl.cpp @@ -1,13 +1,12 @@ #include "common.hpp" #include -static long sys_fcntl(int fd, int cmd, va_list va) { - try { - return FD_map::_get(fd).fcntl(cmd, va); - } - catch(const FD_not_found&) { - return -EBADF; - } +static long sys_fcntl(int fd, int cmd, va_list va) +{ + if(auto* fildes = FD_map::_get(fd); fildes) + return fildes->fcntl(cmd, va); + + return -EBADF; } extern "C" diff --git a/src/musl/fstat.cpp b/src/musl/fstat.cpp index d1523cfd6e..2530cc9bd3 100644 --- a/src/musl/fstat.cpp +++ b/src/musl/fstat.cpp @@ -3,18 +3,15 @@ #include -static long sys_fstat(int filedes, struct stat* stat_buf) +static long sys_fstat(int fd, struct stat* stat_buf) { if (UNLIKELY(stat_buf == nullptr)) return -EINVAL; - try { - auto& fd = FD_map::_get(filedes); - return fd.fstat(stat_buf); - } - catch(const FD_not_found&) { - return -EBADF; - } + if(auto* fildes = FD_map::_get(fd); fildes) + return fildes->fstat(stat_buf); + + return -EBADF; } extern "C" diff --git a/src/musl/fstatat.cpp b/src/musl/fstatat.cpp index 9c6a1324e7..bf9ec58bbe 100644 --- a/src/musl/fstatat.cpp +++ b/src/musl/fstatat.cpp @@ -6,9 +6,9 @@ long sys_getcwd(char *buf, size_t size); long sys_stat(const char *path, struct stat *buf); -static long sys_fstatat(int filedes, const char *path, struct stat *buf, int flag) +static long sys_fstatat(int fd, const char *path, struct stat *buf, int flag) { - if (filedes == AT_FDCWD) + if (fd == AT_FDCWD) { char cwd_buf[PATH_MAX]; char abs_path[PATH_MAX]; @@ -19,13 +19,10 @@ static long sys_fstatat(int filedes, const char *path, struct stat *buf, int fla } else { - try { - auto& fd = FD_map::_get(filedes); - return fd.fstatat(path, buf, flag); - } - catch(const FD_not_found&) { - return -EBADF; - } + if(auto* fildes = FD_map::_get(fd); fildes) + return fildes->fstatat(path, buf, flag); + + return -EBADF; } } diff --git a/src/musl/getdents.cpp b/src/musl/getdents.cpp index 821cd86d94..484a3ded42 100644 --- a/src/musl/getdents.cpp +++ b/src/musl/getdents.cpp @@ -2,13 +2,12 @@ #include #include -static long sys_getdents(unsigned int fd, struct dirent *dirp, unsigned int count) { - try { - return FD_map::_get(fd).getdents(dirp, count); - } - catch(const FD_not_found&) { - return -EBADF; - } +static long sys_getdents(unsigned int fd, struct dirent *dirp, unsigned int count) +{ + if(auto* fildes = FD_map::_get(fd); fildes) + fildes->getdents(dirp, count); + + return -EBADF; } extern "C" diff --git a/src/musl/ioctl.cpp b/src/musl/ioctl.cpp index 13e387ffd2..9b79798a64 100644 --- a/src/musl/ioctl.cpp +++ b/src/musl/ioctl.cpp @@ -20,14 +20,10 @@ static int sys_ioctl(int fd, int req, void* arg) { return 0; } - try { - auto& fildes = FD_map::_get(fd); - return fildes.ioctl(req, arg); - } - catch(const FD_not_found&) { - errno = EBADF; - return -1; - } + if(auto* fildes = FD_map::_get(fd); fildes) + return fildes->ioctl(req, arg); + + return -EBADF; } extern "C" diff --git a/src/musl/lseek.cpp b/src/musl/lseek.cpp index e100c0f1a1..4d1cc02858 100644 --- a/src/musl/lseek.cpp +++ b/src/musl/lseek.cpp @@ -6,14 +6,12 @@ #include "stub.hpp" #include -static off_t sys_lseek(int fd, off_t offset, int whence) { - try { - auto& fildes = FD_map::_get(fd); - return fildes.lseek(offset, whence); - } - catch(const FD_not_found&) { - return -EBADF; - } +static off_t sys_lseek(int fd, off_t offset, int whence) +{ + if(auto* fildes = FD_map::_get(fd); fildes) + return fildes->lseek(offset, whence); + + return -EBADF; } static off_t sys__llseek(unsigned int /*fd*/, unsigned long /*offset_high*/, diff --git a/src/musl/read.cpp b/src/musl/read.cpp index 4c70943284..80dd986dc7 100644 --- a/src/musl/read.cpp +++ b/src/musl/read.cpp @@ -6,13 +6,10 @@ static long sys_read(int fd, void* buf, size_t len) if(UNLIKELY(buf == nullptr)) return -EFAULT; - try { - auto& fildes = FD_map::_get(fd); - return fildes.read(buf, len); - } - catch(const FD_not_found&) { - return -EBADF; - } + if(auto* fildes = FD_map::_get(fd); fildes) + return fildes->read(buf, len); + + return -EBADF; } extern "C" diff --git a/src/musl/readv.cpp b/src/musl/readv.cpp index 9220232546..3b6c3e5edc 100644 --- a/src/musl/readv.cpp +++ b/src/musl/readv.cpp @@ -4,13 +4,10 @@ static ssize_t sys_readv(int fd, const struct iovec* iov, int iovcnt) { - try { - auto& fildes = FD_map::_get(fd); - return fildes.readv(iov, iovcnt); - } - catch(const FD_not_found&) { - return -EBADF; - } + if(auto* fildes = FD_map::_get(fd); fildes) + return fildes->readv(iov, iovcnt); + + return -EBADF; } extern "C" diff --git a/src/musl/socketcall.cpp b/src/musl/socketcall.cpp index 2c5b35564e..5d8f4a7998 100644 --- a/src/musl/socketcall.cpp +++ b/src/musl/socketcall.cpp @@ -34,60 +34,45 @@ static long sock_socket(int domain, int type, int protocol) static long sock_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) { - try { - auto& fildes = FD_map::_get(sockfd); - return fildes.connect(addr, addrlen); - } - catch(const FD_not_found&) { - return -EBADF; - } + if(auto* fildes = FD_map::_get(sockfd); fildes) + return fildes->connect(addr, addrlen); + + return -EBADF; } static long sock_bind(int sockfd, const struct sockaddr *addr, socklen_t addrlen) { - try { - auto& fildes = FD_map::_get(sockfd); - return fildes.connect(addr, addrlen); - } - catch(const FD_not_found&) { - return -EBADF; - } + if(auto* fildes = FD_map::_get(sockfd); fildes) + return fildes->bind(addr, addrlen); + + return -EBADF; } static long sock_sendmsg(int sockfd, const struct msghdr *msg, int flags) { - try { - auto& fildes = FD_map::_get(sockfd); - return fildes.sendmsg(msg, flags); - } - catch(const FD_not_found&) { - return -EBADF; - } + if(auto* fildes = FD_map::_get(sockfd); fildes) + return fildes->sendmsg(msg, flags); + + return -EBADF; } static ssize_t sock_sendto(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen) { - try { - auto& fildes = FD_map::_get(sockfd); - return fildes.sendto(buf, len, flags, dest_addr, addrlen); - } - catch(const FD_not_found&) { - return -EBADF; - } + if(auto* fildes = FD_map::_get(sockfd); fildes) + return fildes->sendto(buf, len, flags, dest_addr, addrlen); + + return -EBADF; } static ssize_t sock_recvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen) { - try { - auto& fildes = FD_map::_get(sockfd); - return fildes.recvfrom(buf, len, flags, src_addr, addrlen); - } - catch(const FD_not_found&) { - return -EBADF; - } + if(auto* fildes = FD_map::_get(sockfd); fildes) + return fildes->recvfrom(buf, len, flags, src_addr, addrlen); + + return -EBADF; } extern "C" diff --git a/src/musl/write.cpp b/src/musl/write.cpp index 84a3ed495f..e044c3d48f 100644 --- a/src/musl/write.cpp +++ b/src/musl/write.cpp @@ -11,13 +11,10 @@ static long sys_write(int fd, char* str, size_t len) { return len; } - try { - auto& fildes = FD_map::_get(fd); - return fildes.write(str, len); - } - catch(const FD_not_found&) { - return -EBADF; - } + if(auto* fildes = FD_map::_get(fd); fildes) + return fildes->write(str, len); + + return -EBADF; } // The syscall wrapper, using strace if enabled From bd1fa3e60fa0f1be27d1918e503561ec8888b1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 6 Apr 2018 11:56:03 +0200 Subject: [PATCH 129/723] protobuf: Silence warnings --- cmake/protobuf.cmake | 1 + 1 file changed, 1 insertion(+) diff --git a/cmake/protobuf.cmake b/cmake/protobuf.cmake index 612653e244..bb04131ed7 100644 --- a/cmake/protobuf.cmake +++ b/cmake/protobuf.cmake @@ -99,6 +99,7 @@ set(PROTOBUF_SOURCES add_library(protobuf STATIC ${PROTOBUF_SOURCES}) target_compile_definitions(protobuf PRIVATE HAVE_PTHREAD=0) +target_compile_options(protobuf PRIVATE -Wno-sign-compare -Wno-unused-parameter) # Make sure precompiled libraries exists add_dependencies(protobuf PrecompiledLibraries) From 677637cad9bec5308b25629862a0662ce03b4560 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 6 Apr 2018 14:14:12 +0200 Subject: [PATCH 130/723] bundle: add precompiled libgcc --- cmake/cross_compiled_libraries.cmake | 32 ++++++---------------------- cmake/post.service.cmake | 10 ++++++--- etc/create_binary_bundle.sh | 31 +++++++++++++++++++++------ 3 files changed, 37 insertions(+), 36 deletions(-) diff --git a/cmake/cross_compiled_libraries.cmake b/cmake/cross_compiled_libraries.cmake index 98186a0433..b7d6d9e506 100644 --- a/cmake/cross_compiled_libraries.cmake +++ b/cmake/cross_compiled_libraries.cmake @@ -62,32 +62,15 @@ endif (WITH_SOLO5) set(PRECOMPILED_DIR ${CMAKE_CURRENT_BINARY_DIR}/precompiled/src/PrecompiledLibraries/${ARCH}) +set(LIBGCC_LIB_DIR ${PRECOMPILED_DIR}/libgcc/) set(LIBCXX_INCLUDE_DIR ${PRECOMPILED_DIR}/libcxx/include/) set(LIBCXX_LIB_DIR ${PRECOMPILED_DIR}/libcxx/) set(LIBUNWIND_INCLUDE_DIR ${PRECOMPILED_DIR}/libunwind/include/) set(LIBUNWIND_LIB_DIR ${PRECOMPILED_DIR}/libunwind/) -add_library(libcxx STATIC IMPORTED) -add_library(libcxxabi STATIC IMPORTED) -add_library(libunwind STATIC IMPORTED) - -add_dependencies(libcxx PrecompiledLibraries) -add_dependencies(libcxxabi PrecompiledLibraries) -set_target_properties(libcxx PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libc++.a) -set_target_properties(libcxxabi PROPERTIES IMPORTED_LOCATION ${LIBCXX_LIB_DIR}/libc++abi.a) -set_target_properties(libunwind PROPERTIES IMPORTED_LOCATION ${LIBUNWIND_LIB_DIR}/libunwind.a) - set(MUSL_INCLUDE_DIR ${PRECOMPILED_DIR}/musl/include/) set(MUSL_LIB_DIR ${PRECOMPILED_DIR}/musl/lib) -add_library(libc STATIC IMPORTED) -set_target_properties(libc PROPERTIES IMPORTED_LOCATION ${MUSL_LIB_DIR}/libc.a) -add_dependencies(libc PrecompiledLibraries) - -add_library(libm STATIC IMPORTED) -set_target_properties(libm PROPERTIES IMPORTED_LOCATION ${MUSL_LIB_DIR}/libm.a) -add_dependencies(libm PrecompiledLibraries) - # # Installation # @@ -97,12 +80,6 @@ install(DIRECTORY ${LIBUNWIND_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include install(DIRECTORY ${MUSL_INCLUDE_DIR} DESTINATION includeos/${ARCH}/include/musl) -set(CHAINLOAD_LOC ${CMAKE_CURRENT_BINARY_DIR}/precompiled/src/PrecompiledLibraries/chainloader) -install(FILES ${CHAINLOAD_LOC} DESTINATION includeos) - -file(GLOB musl_libs ${MUSL_LIB_DIR}/*.a) -file(GLOB musl_objs ${MUSL_LIB_DIR}/*.o) - add_custom_command(TARGET PrecompiledLibraries POST_BUILD COMMAND ${CMAKE_COMMAND} -E echo "Installed elf.h into ${CMAKE_INSTALL_PREFIX}/include" ) @@ -113,11 +90,14 @@ ExternalProject_Add_Step(PrecompiledLibraries copy_elf DEPENDEES download ) +# Install musl +install(DIRECTORY ${MUSL_LIB_DIR}/ DESTINATION includeos/${ARCH}/lib) + +# Install libc++ etc. install(FILES - ${musl_libs} - ${musl_objs} ${LIBCXX_LIB_DIR}/libc++.a ${LIBCXX_LIB_DIR}/libc++abi.a + ${LIBGCC_LIB_DIR}/libcompiler.a ${LIBUNWIND_LIB_DIR}/libunwind.a DESTINATION includeos/${ARCH}/lib) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 09446ca56f..413d4e696c 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -8,7 +8,6 @@ if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) endif() set(INSTALL_LOC $ENV{INCLUDEOS_PREFIX}/includeos) -set(INSTALL_LOC_LIBGCC $ENV{INCLUDEOS_PREFIX}/includeos) message(STATUS "Target triple ${TRIPLE}") @@ -266,7 +265,7 @@ if ("${PLATFORM}" STREQUAL "x86_solo5") set(PRE_BSS_SIZE "--defsym PRE_BSS_AREA=0x200000") endif() -set(LDFLAGS "-nostdlib -melf_${ELF} --eh-frame-hdr ${STRIP_LV} --script=${INSTALL_LOC}/${ARCH}/linker.ld ${PRE_BSS_SIZE}") +set(LDFLAGS "-nostdlib -melf_${ELF} --eh-frame-hdr ${STRIP_LV} --script=${INSTALL_LOC}/${ARCH}/linker.ld ${PRE_BSS_SIZE}") set_target_properties(service PROPERTIES LINK_FLAGS "${LDFLAGS}") @@ -276,6 +275,11 @@ set(CRTI "${INSTALL_LOC}/${ARCH}/lib/crti.o") target_link_libraries(service ${CRTI}) target_link_libraries(service ${CRT1}) +add_library(libgcc STATIC IMPORTED) +set_target_properties(libgcc PROPERTIES LINKER_LANGUAGE C) +set_target_properties(libgcc PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libcompiler.a) + + add_library(libos STATIC IMPORTED) set_target_properties(libos PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(libos PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libos.a) @@ -459,7 +463,7 @@ target_link_libraries(service libos libcxx libc - + libgcc ${CRTN} ) # write binary location to known file diff --git a/etc/create_binary_bundle.sh b/etc/create_binary_bundle.sh index 859c92f6f1..0afd8d43bb 100755 --- a/etc/create_binary_bundle.sh +++ b/etc/create_binary_bundle.sh @@ -22,11 +22,12 @@ export llvm_branch=${llvm_branch:-release_50} # Options to skip steps [ ! -v do_binutils ] && do_binutils=1 [ ! -v do_musl ] && do_musl=1 -[ ! -v do_libunwind ] && do_libunwind=1; +[ ! -v do_libunwind ] && do_libunwind=1 [ ! -v do_includeos ] && do_includeos=1 [ ! -v do_llvm ] && do_llvm=1 [ ! -v do_bridge ] && do_bridge=1 [ ! -v do_packages ] && do_packages=1 +[ ! -v do_build ] && do_build=1 ############################################################ # COMMAND LINE PROPERTIES: @@ -105,6 +106,15 @@ DIR_NAME="IncludeOS_dependencies" OUTFILE="${DIR_NAME}_$filename_tag.tar.gz" BUNDLE_PATH=${BUNDLE_PATH:-$BUILD_DIR} +function export_libgcc { + if [ $ARCH == i686 ] + then + export LGCC_ARCH_PARAM="-m32" + fi + export libgcc=$($CC $LGCC_ARCH_PARAM "--print-libgcc-file-name") + echo ">>> Copying libgcc from: $libgcc" +} + function do_build { echo -e "\n\n >>> Building bundle for ${ARCH} \n" # Build all sources @@ -134,6 +144,7 @@ function do_build { musl=$TEMP_INSTALL_DIR/$TARGET/lib llvm=$BUILD_DIR/build_llvm + libcpp=$llvm/lib/libc++.a libunwind=$llvm/lib/libunwind.a libcppabi=$llvm/lib/libc++abi.a @@ -146,6 +157,7 @@ function do_build { # Make directory-tree mkdir -p $BUNDLE_LOC mkdir -p $BUNDLE_LOC/musl + mkdir -p $BUNDLE_LOC/libgcc mkdir -p $BUNDLE_LOC/libcxx mkdir -p $BUNDLE_LOC/libunwind mkdir -p $BUNDLE_LOC/libunwind/include @@ -155,6 +167,7 @@ function do_build { cp $libcppabi $BUNDLE_LOC/libcxx/ cp $libunwind $BUNDLE_LOC/libunwind/ cp -r $musl $BUNDLE_LOC/musl/ + cp $libgcc $BUNDLE_LOC/libgcc/libcompiler.a # Copy includes cp -r $include_musl $BUNDLE_LOC/musl/ @@ -164,12 +177,16 @@ function do_build { } -for B_ARCH in $BUNDLE_ARCHES -do - export ARCH=$B_ARCH - export TARGET=$ARCH-elf # Configure target based on arch. Always ELF. - do_build -done + +if [ ! -z $do_build ]; then + for B_ARCH in $BUNDLE_ARCHES + do + export ARCH=$B_ARCH + export_libgcc + export TARGET=$ARCH-elf # Configure target based on arch. Always ELF. + do_build + done +fi # Zip it tar -czvf $OUTFILE --directory=$BUNDLE_PATH $DIR_NAME From c1e5ebd6127f6c6ae2004ddb3d2c644a4944421e Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 6 Apr 2018 14:33:14 +0200 Subject: [PATCH 131/723] chainload: supress stdout by default --- src/chainload/CMakeLists.txt | 2 ++ src/platform/x86_nano/kernel_start.cpp | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/chainload/CMakeLists.txt b/src/chainload/CMakeLists.txt index fcda18daa2..daadf33d8f 100644 --- a/src/chainload/CMakeLists.txt +++ b/src/chainload/CMakeLists.txt @@ -7,6 +7,8 @@ endif() set(ARCH i686) set(PLATFORM x86_nano) +option(default_stdout "" OFF) + include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) project (chainloader) diff --git a/src/platform/x86_nano/kernel_start.cpp b/src/platform/x86_nano/kernel_start.cpp index 04bf5958cc..775838f816 100644 --- a/src/platform/x86_nano/kernel_start.cpp +++ b/src/platform/x86_nano/kernel_start.cpp @@ -30,7 +30,8 @@ extern "C" { void _init_heap(uintptr_t); void _init_syscalls(); } -bool __libc_initialized = false; +bool __libc_initialized = true; +extern bool os_default_stdout; extern "C" void kernel_start(uintptr_t magic, uintptr_t addr) @@ -60,7 +61,8 @@ void kernel_start(uintptr_t magic, uintptr_t addr) _init_syscalls(); // Initialize stdout handlers - OS::add_stdout(&OS::default_stdout); + if (os_default_stdout) + OS::add_stdout(&OS::default_stdout); OS::start(magic, addr); From cd0ef554760ad6570135d4c1e466d01a6aaa3f11 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 6 Apr 2018 14:39:41 +0200 Subject: [PATCH 132/723] musl/common: avoid __assert_fail, stubtrace not using printf --- api/common | 6 +++--- api/info | 2 +- src/musl/stub.hpp | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/api/common b/api/common index 91e8c487eb..25663a5792 100644 --- a/api/common +++ b/api/common @@ -57,7 +57,7 @@ static_assert(sizeof(void*) == 8, "Pointer must match arch"); #include #endif -inline void __assert_fail(const char *expr, const char *file, int line, const char *func){ +inline void __expect_fail(const char *expr, const char *file, int line, const char *func){ #ifndef INCLUDEOS_SINGLE_THREADED SMP::global_lock(); #endif @@ -69,8 +69,8 @@ inline void __assert_fail(const char *expr, const char *file, int line, const ch exit(67); } -#define Expects(x) ((void)((x) || (__assert_fail("Expects failed: "#x, __FILE__, __LINE__, __func__),0))) -#define Ensures(x) ((void)((x) || (__assert_fail("Ensures failed: "#x, __FILE__, __LINE__, __func__),0))) +#define Expects(x) ((void)((x) || (__expect_fail("Expects failed: "#x, __FILE__, __LINE__, __func__),0))) +#define Ensures(x) ((void)((x) || (__expect_fail("Ensures failed: "#x, __FILE__, __LINE__, __func__),0))) #endif //< defined(OS_TERMINATE_ON_CONTRACT_VIOLATION) diff --git a/api/info b/api/info index a0e5b694af..c907737b6d 100644 --- a/api/info +++ b/api/info @@ -32,7 +32,7 @@ // Checkboxes - for initialization output and testing #define CHECK(TEST, TEXT, ...) printf("%16s[%s] " TEXT "\n","", TEST ? "x" : " ", ##__VA_ARGS__) -#define CHECKSERT(TEST, TEXT, ...) assert(TEST), CHECK(TEST, TEXT, ##__VA_ARGS__) +#define CHECKSERT(TEST, TEXT, ...) CHECK(TEST, TEXT, ##__VA_ARGS__), assert(TEST) #define FAIL(TEXT, ...) printf("FAIL: " TEXT "\n", ##__VA_ARGS__); panic("FAIL") // Undefine diff --git a/src/musl/stub.hpp b/src/musl/stub.hpp index 1b61f78b2d..d1ecaae5e3 100644 --- a/src/musl/stub.hpp +++ b/src/musl/stub.hpp @@ -1,8 +1,11 @@ #include "common.hpp" +extern "C" +void __serial_print1(const char*); + template inline void stubtrace_print(const char* name, R ret, Args&&... args) { - printf(" Stubbed syscall: "); + __serial_print1(" Stubbed syscall: "); strace_print(name, ret, args...); } From cc1ddb0b69f87c2ac2ed4c87902ee25f0e0207d4 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 6 Apr 2018 14:46:09 +0200 Subject: [PATCH 133/723] musl: change socketcall to be syscall compatible --- etc/musl/syscall.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etc/musl/syscall.h b/etc/musl/syscall.h index 525232f018..705d8c352a 100644 --- a/etc/musl/syscall.h +++ b/etc/musl/syscall.h @@ -47,9 +47,8 @@ long __syscall_ret(unsigned long), __syscall(syscall_arg_t, ...), #define __syscall_cp syscall #define syscall_cp syscall -#define __socketcall(nm,a,b,c,d,e,f) socketcall_##nm \ - ((long [6]){ (long)a, (long)b, (long)c, (long)d, (long)e, (long)f }) -#define __socketcall_cp(nm,a,b,c,d,e,f) __syscall_ret(__socketcall(nm,a,b,c,d,e,f)) +#define __socketcall(nm, ...) socketcall_##nm(__VA_ARGS__) +#define __socketcall_cp(nm, ...) __syscall_ret(socketcall_##nm(__VA_ARGS__)) /* fixup legacy 16-bit junk */ From aedd14be433c63e94c8b38610a0622add6c08bac Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 6 Apr 2018 14:59:11 +0200 Subject: [PATCH 134/723] cmake: update musl bundle --- cmake/cross_compiled_libraries.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/cross_compiled_libraries.cmake b/cmake/cross_compiled_libraries.cmake index b7d6d9e506..256f3237d9 100644 --- a/cmake/cross_compiled_libraries.cmake +++ b/cmake/cross_compiled_libraries.cmake @@ -18,8 +18,8 @@ else(BUNDLE_LOC) include(ExternalProject) ExternalProject_Add(PrecompiledLibraries PREFIX precompiled - URL https://github.com/hioa-cs/IncludeOS/releases/download/v0.12.0-rc.2/IncludeOS_dependencies_v0-12-0_musl_libunwind_threaded.tar.gz - URL_HASH SHA1=5a9daff15ed7b0fcd2a96f3c07ebef5aabf93371 + URL https://github.com/hioa-cs/IncludeOS/releases/download/v0.12.0-rc.2/IncludeOS_dependencies_v0-12-0_musl_libunwind_singlethreaded.tar.gz + URL_HASH SHA1=febb731b35130431871d46476ce673781e9ce8db CONFIGURE_COMMAND "" BUILD_COMMAND "" UPDATE_COMMAND "" From 5dc539aefc01ec648467617a88ab44202aaf66bd Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 6 Apr 2018 15:12:35 +0200 Subject: [PATCH 135/723] test: fix fd_map test --- api/posix/fd_map.hpp | 1 + test/posix/unit/fd_map_test.cpp | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api/posix/fd_map.hpp b/api/posix/fd_map.hpp index ab1a33b825..4adb95e802 100644 --- a/api/posix/fd_map.hpp +++ b/api/posix/fd_map.hpp @@ -24,6 +24,7 @@ #include #include #include +#include class FD_map_error : public std::runtime_error { using std::runtime_error::runtime_error; diff --git a/test/posix/unit/fd_map_test.cpp b/test/posix/unit/fd_map_test.cpp index 872b48695d..610a987694 100644 --- a/test/posix/unit/fd_map_test.cpp +++ b/test/posix/unit/fd_map_test.cpp @@ -23,7 +23,7 @@ class Test_fd : public FD { public: Test_fd(const int id) : FD(id) {}; - int read(void*, size_t) override + ssize_t read(void*, size_t) override { return 1; } int close() override @@ -57,11 +57,12 @@ CASE("Adding a implemented FD descriptor in FD_map") // Get works const FD_map::id_t id = test.get_id(); - auto& get = FD_map::_get(id); - EXPECT(get == test); + auto* get = FD_map::_get(id); + EXPECT(get != nullptr); + EXPECT(*get == test); // Close works - FD_map::_close(id); + FD_map::close(id); // Throws when not found works EXPECT_THROWS_AS(FD_map::_get(id), FD_not_found); From 5da10232be42469097055923ce8d07743b409c31 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Sat, 7 Apr 2018 08:34:44 +0200 Subject: [PATCH 136/723] test: remove kprintf test --- test/CMakeLists.txt | 2 -- test/kernel/unit/kprint_test.cpp | 31 ------------------------------- test/lest_util/os_mock.cpp | 11 +++++++---- 3 files changed, 7 insertions(+), 37 deletions(-) delete mode 100644 test/kernel/unit/kprint_test.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 063c28d6b0..b5768b8bb5 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -76,7 +76,6 @@ set(TEST_SOURCES ${TEST}/hw/unit/mac_addr_test.cpp ${TEST}/hw/unit/nic_test.cpp ${TEST}/kernel/unit/events_test.cpp - ${TEST}/kernel/unit/kprint_test.cpp ${TEST}/kernel/unit/memmap_test.cpp ${TEST}/kernel/unit/memory.cpp ${TEST}/kernel/unit/x86_paging.cpp @@ -152,7 +151,6 @@ set(OS_SOURCES ${SRC}/hw/pci_device.cpp ${SRC}/hw/pci_msi.cpp ${SRC}/hw/ps2.cpp - ${SRC}/hw/serial.cpp ${SRC}/kernel/cpuid.cpp ${SRC}/kernel/elf.cpp ${SRC}/kernel/events.cpp diff --git a/test/kernel/unit/kprint_test.cpp b/test/kernel/unit/kprint_test.cpp deleted file mode 100644 index dfaa9fc9c1..0000000000 --- a/test/kernel/unit/kprint_test.cpp +++ /dev/null @@ -1,31 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2016-2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include - -static char buf[1024]; - -void __serial_print1(const char* cstr) { - snprintf(buf, 1024, "%s", cstr); -} - -CASE("kprintf() prints formatted strings") -{ - kprintf("Error code: 0x%x \n", 1); - EXPECT(strcmp(buf, "Error code: 0x1 \n") == 0); -} diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index 08a90f995e..c706162091 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -156,13 +156,16 @@ extern "C" { return {0}; } void malloc_trim() {} + uintptr_t heap_end = std::numeric_limits::max(); - __attribute__((weak)) void __init_serial1 () {} - __attribute__((weak)) + void __serial_print1(const char* cstr) { - static char __printbuf[4096]; - snprintf(__printbuf, sizeof(__printbuf), "%s", cstr); + printf(" %s\n", cstr); + } + + void __serial_print(const char* cstr, int len) { + printf(" %.*s", len, cstr); } } // ~ extern "C" From b8783a77ef055315cd7973e277a4d0525e3c9b67 Mon Sep 17 00:00:00 2001 From: nikhilap1 Date: Sun, 8 Apr 2018 18:31:15 +0530 Subject: [PATCH 137/723] Replace OS::nanos_since_boot() with RTC::nanos_now() to the get the timing working with solo5 --- examples/UDP_perf/service.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/UDP_perf/service.cpp b/examples/UDP_perf/service.cpp index 0a96985be4..5b3dc72cb2 100644 --- a/examples/UDP_perf/service.cpp +++ b/examples/UDP_perf/service.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #define CLIENT_PORT 1337 #define SERVER_PORT 1338 @@ -73,7 +74,7 @@ void init_sample_stats() initial_packets_tx = Statman::get().get_by_name("eth0.ethernet.packets_tx").get_uint64(); prev_packets_rx = initial_packets_rx; prev_packets_tx = initial_packets_tx; - first_ts = OS::nanos_since_boot(); + first_ts = RTC::nanos_now(); sample_ts = last_ts = first_ts; activity_before.reset(); } @@ -122,7 +123,7 @@ void send_data(net::UDPSocket& client, net::Inet& inet) { client.sendto(inet.gateway(), NCAT_RECEIVE_PORT, buff.data(), buff.size(), send_cb); } sample_ts = last_ts; - last_ts = OS::nanos_since_boot(); + last_ts = RTC::nanos_now(); printf("Done sending data\n"); } void Service::start(const std::string& input) { @@ -176,7 +177,7 @@ void Service::start(const std::string& input) { data_len += data.size(); data_received = true; sample_ts = last_ts; - last_ts = OS::nanos_since_boot(); + last_ts = RTC::nanos_now(); }); Timers::periodic(5s, 5s, From c89ea19e82a5e24572745dcb9d07a7b21569d4a9 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 9 Apr 2018 10:35:27 +0200 Subject: [PATCH 138/723] kernel: Move OS::block from platform to kernel --- src/CMakeLists.txt | 2 +- src/{platform/x86_pc => kernel}/block.cpp | 0 src/platform/x86_pc/CMakeLists.txt | 1 - 3 files changed, 1 insertion(+), 2 deletions(-) rename src/{platform/x86_pc => kernel}/block.cpp (100%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 15a0f0aa8d..90f394f9fd 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -29,7 +29,7 @@ set(BOTAN_MODULES "net/https/botan_server.cpp") set(OS_OBJECTS kernel/multiboot.cpp - kernel/syscalls.cpp kernel/os.cpp kernel/cpuid.cpp + kernel/syscalls.cpp kernel/os.cpp kernel/cpuid.cpp kernel/block.cpp kernel/events.cpp kernel/memmap.cpp kernel/pci_manager.cpp kernel/heap.cpp kernel/service_stub.cpp kernel/elf.cpp kernel/vga.cpp kernel/context.cpp kernel/context_asm.asm diff --git a/src/platform/x86_pc/block.cpp b/src/kernel/block.cpp similarity index 100% rename from src/platform/x86_pc/block.cpp rename to src/kernel/block.cpp diff --git a/src/platform/x86_pc/CMakeLists.txt b/src/platform/x86_pc/CMakeLists.txt index 21193e5231..0ec8a61b45 100644 --- a/src/platform/x86_pc/CMakeLists.txt +++ b/src/platform/x86_pc/CMakeLists.txt @@ -4,7 +4,6 @@ set(X86_PC_OBJECTS kernel_start.cpp platform.cpp os.cpp - block.cpp clocks.cpp cmos.cpp cmos_clock.cpp From 66f312b192bba7b40f1bb8b03c117da66fecf21a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 9 Apr 2018 10:40:17 +0200 Subject: [PATCH 139/723] musl: Readded TCP & UDP fd, removed posix include --- api/posix/tcp_fd.hpp | 2 +- api/posix/udp_fd.hpp | 3 +- cmake/post.service.cmake | 1 - src/CMakeLists.txt | 3 +- src/musl/ioctl.cpp | 2 +- src/musl/read.cpp | 2 +- src/musl/socketcall.cpp | 10 +- src/platform/x86_pc/pic.hpp | 2 +- src/plugins/vfs.cpp | 2 +- src/posix/tcp_fd.cpp | 384 +++++++++++++++++++++++++++++++++++ src/posix/udp_fd.cpp | 387 ++++++++++++++++++++++++++++++++++++ 11 files changed, 784 insertions(+), 14 deletions(-) create mode 100644 src/posix/tcp_fd.cpp create mode 100644 src/posix/udp_fd.cpp diff --git a/api/posix/tcp_fd.hpp b/api/posix/tcp_fd.hpp index 0074fffcd3..5b7bb4cdc4 100644 --- a/api/posix/tcp_fd.hpp +++ b/api/posix/tcp_fd.hpp @@ -33,7 +33,7 @@ class TCP_FD : public SockFD { : SockFD(id) {} - int read(void*, size_t) override; + ssize_t read(void*, size_t) override; int write(const void*, size_t) override; int close() override; diff --git a/api/posix/udp_fd.hpp b/api/posix/udp_fd.hpp index 17c0e3aa87..8e9a0dd169 100644 --- a/api/posix/udp_fd.hpp +++ b/api/posix/udp_fd.hpp @@ -23,6 +23,7 @@ #include #include #include +#include class UDP_FD : public SockFD { public: @@ -39,7 +40,7 @@ class UDP_FD : public SockFD { ~UDP_FD(); - int read(void*, size_t) override; + ssize_t read(void*, size_t) override; int write(const void*, size_t) override; int close() override; diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 413d4e696c..b382468fe8 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -225,7 +225,6 @@ endforeach() # includes include_directories(${LOCAL_INCLUDES}) -include_directories(${INSTALL_LOC}/api/posix) include_directories(${INSTALL_LOC}/${ARCH}/include/libcxx) include_directories(${INSTALL_LOC}/${ARCH}/include/musl) include_directories(${INSTALL_LOC}/${ARCH}/include/libunwind) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 15a0f0aa8d..3aae160ea3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,7 +7,6 @@ if (NOT threading) add_definitions(-DINCLUDEOS_SINGLE_THREADED) endif() -include_directories(${INCLUDEOS_ROOT}/api/posix) include_directories(${LIBCXX_INCLUDE_DIR}) include_directories(${MUSL_INCLUDE_DIR}) include_directories(${SOLO5_INCLUDE_DIR}) @@ -61,7 +60,7 @@ set(OS_OBJECTS fs/disk.cpp fs/filesystem.cpp fs/dirent.cpp fs/mbr.cpp fs/path.cpp fs/fat.cpp fs/fat_async.cpp fs/fat_sync.cpp fs/memdisk.cpp # POSIX - posix/fd.cpp posix/file_fd.cpp + posix/fd.cpp posix/file_fd.cpp posix/tcp_fd.cpp posix/udp_fd.cpp ) add_library(os STATIC ${OS_OBJECTS}) diff --git a/src/musl/ioctl.cpp b/src/musl/ioctl.cpp index 9b79798a64..5146cbf7f1 100644 --- a/src/musl/ioctl.cpp +++ b/src/musl/ioctl.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include "stub.hpp" diff --git a/src/musl/read.cpp b/src/musl/read.cpp index 80dd986dc7..07f24d272e 100644 --- a/src/musl/read.cpp +++ b/src/musl/read.cpp @@ -1,5 +1,5 @@ #include "common.hpp" -#include +#include static long sys_read(int fd, void* buf, size_t len) { diff --git a/src/musl/socketcall.cpp b/src/musl/socketcall.cpp index 5d8f4a7998..024e54de9d 100644 --- a/src/musl/socketcall.cpp +++ b/src/musl/socketcall.cpp @@ -3,6 +3,8 @@ #include #include +#include +#include static long sock_socket(int domain, int type, int protocol) { @@ -16,7 +18,7 @@ static long sock_socket(int domain, int type, int protocol) if (UNLIKELY(protocol < 0)) return -EPROTONOSUPPORT; - /*return [](const int type)->int{ + return [](const int type)->int{ switch(type) { case SOCK_STREAM: @@ -24,11 +26,9 @@ static long sock_socket(int domain, int type, int protocol) case SOCK_DGRAM: return FD_map::_open().get_id(); default: - errno = EINVAL; - return -1; + return -EINVAL; } - }(type);*/ - return -EINVAL; + }(type); } static long sock_connect(int sockfd, const struct sockaddr *addr, diff --git a/src/platform/x86_pc/pic.hpp b/src/platform/x86_pc/pic.hpp index 5f22ff402d..13abccedaf 100644 --- a/src/platform/x86_pc/pic.hpp +++ b/src/platform/x86_pc/pic.hpp @@ -18,7 +18,7 @@ #ifndef X86_PIC_HPP #define X86_PIC_HPP -#include "../kernel/os.hpp" +#include #include #include diff --git a/src/plugins/vfs.cpp b/src/plugins/vfs.cpp index f085dcb017..caff2ebff5 100644 --- a/src/plugins/vfs.cpp +++ b/src/plugins/vfs.cpp @@ -104,7 +104,7 @@ static void parse_config() } -#include +#include // Mount RNG functionality to system paths static void mount_rng() { diff --git a/src/posix/tcp_fd.cpp b/src/posix/tcp_fd.cpp new file mode 100644 index 0000000000..77b3c681cd --- /dev/null +++ b/src/posix/tcp_fd.cpp @@ -0,0 +1,384 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017-2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +#define POSIX_STRACE +#ifdef POSIX_STRACE +#define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define PRINT(fmt, ...) /* fmt */ +#endif + +using namespace net; + +// return the "currently selected" networking stack +static auto& net_stack() { + return Inet4::stack(); +} + +ssize_t TCP_FD::read(void* data, size_t len) +{ + return recv(data, len, 0); +} +int TCP_FD::write(const void* data, size_t len) +{ + return send(data, len, 0); +} + +int TCP_FD::close() +{ + // connection + if (this->cd) { + PRINT("TCP: close(%s)\n", cd->to_string().c_str()); + int ret = cd->close(); + delete cd; cd = nullptr; + return ret; + } + // listener + if (this->ld) { + PRINT("TCP: close(%s)\n", ld->to_string().c_str()); + int ret = ld->close(); + delete ld; ld = nullptr; + return ret; + } + errno = EBADF; + return -1; +} + +int TCP_FD::connect(const struct sockaddr* address, socklen_t address_len) +{ + if (is_listener()) { + PRINT("TCP: connect(%s) failed\n", ld->to_string().c_str()); + // already listening on port + errno = EINVAL; + return -1; + } + if (this->cd) { + PRINT("TCP: connect(%s) failed\n", cd->to_string().c_str()); + // if its straight-up connected, return that + if (cd->conn->is_connected()) { + errno = EISCONN; + return -1; + } + // if the connection isn't closed, we can just assume its being used already + if (!cd->conn->is_closed()) { + errno = EALREADY; + return -1; + } + } + + if (address_len != sizeof(sockaddr_in)) { + errno = EINVAL; // checkme? + return -1; + } + auto* inaddr = (sockaddr_in*) address; + + auto addr = ip4::Addr(inaddr->sin_addr.s_addr); + auto port = ::htons(inaddr->sin_port); + + PRINT("TCP: connect(%s:%u)\n", addr.to_string().c_str(), port); + + auto outgoing = net_stack().tcp().connect({addr, port}); + // O_NONBLOCK is set for the file descriptor for the socket and the connection + // cannot be immediately established; the connection shall be established asynchronously. + if (this->is_blocking() == false) { + errno = EINPROGRESS; + return -1; + } + + // wait for connection state to change + while (not (outgoing->is_connected() || outgoing->is_closing() || outgoing->is_closed())) { + OS::block(); + } + // set connection whether good or bad + if (outgoing->is_connected()) { + // out with the old + delete this->cd; + // in with the new + this->cd = new TCP_FD_Conn(outgoing); + cd->set_default_read(); + return 0; + } + // failed to connect + // TODO: try to distinguish the reason for connection failure + errno = ECONNREFUSED; + return -1; +} + + +ssize_t TCP_FD::send(const void* data, size_t len, int fmt) +{ + if (!cd) { + errno = EINVAL; + return -1; + } + return cd->send(data, len, fmt); +} +ssize_t TCP_FD::recv(void* dest, size_t len, int flags) +{ + if (!cd) { + errno = EINVAL; + return -1; + } + return cd->recv(dest, len, flags); +} + +int TCP_FD::accept(struct sockaddr *__restrict__ addr, socklen_t *__restrict__ addr_len) +{ + if (!ld) { + errno = EINVAL; + return -1; + } + return ld->accept(addr, addr_len); +} +int TCP_FD::listen(int backlog) +{ + if (!ld) { + errno = EINVAL; + return -1; + } + return ld->listen(backlog); +} +int TCP_FD::bind(const struct sockaddr *addr, socklen_t addrlen) +{ + // + if (cd) { + errno = EINVAL; + return -1; + } + // verify socket address + if (addrlen != sizeof(sockaddr_in)) { + errno = EINVAL; + return -1; + } + auto* sin = (sockaddr_in*) addr; + // verify its AF_INET + if (sin->sin_family != AF_INET) { + errno = EAFNOSUPPORT; + return -1; + } + // use sin_port for bind + // its network order ... so swap that shit: + uint16_t port = ::htons(sin->sin_port); + // ignore IP address (FIXME?) + /// TODO: verify that the IP is "local" + try { + auto& L = net_stack().tcp().listen(port); + // remove existing listener + if (ld) { + int ret = ld->close(); + if (ret < 0) return ret; + delete ld; + } + // create new one + ld = new TCP_FD_Listen(L); + return 0; + + } catch (...) { + errno = EADDRINUSE; + return -1; + } +} +int TCP_FD::shutdown(int mode) +{ + if (!cd) { + errno = EINVAL; + return -1; + } + return cd->shutdown(mode); +} + +/// socket default handler getters + +TCP_FD::on_read_func TCP_FD::get_default_read_func() +{ + if (cd) { + return {cd, &TCP_FD_Conn::recv_to_ringbuffer}; + } + if (ld) { + return + [/*this*/] (net::tcp::buffer_t) { + // what to do here? + }; + } + throw std::runtime_error("Invalid socket"); +} +TCP_FD::on_write_func TCP_FD::get_default_write_func() +{ + return [] {}; +} +TCP_FD::on_except_func TCP_FD::get_default_except_func() +{ + return [] {}; +} + +/// socket as connection + +void TCP_FD_Conn::recv_to_ringbuffer(net::tcp::buffer_t buffer) +{ + if (readq.free_space() < (ssize_t) buffer->size()) { + // make room for data + int needed = buffer->size() - readq.free_space(); + int discarded = readq.discard(needed); + assert(discarded == needed); + } + // add data to ringbuffer + int written = readq.write(buffer->data(), buffer->size()); + assert(written == (ssize_t) buffer->size()); +} +void TCP_FD_Conn::set_default_read() +{ + // readq buffering (4kb at a time) + conn->on_read(4096, {this, &TCP_FD_Conn::recv_to_ringbuffer}); +} +ssize_t TCP_FD_Conn::send(const void* data, size_t len, int) +{ + if (not conn->is_connected()) { + errno = ENOTCONN; + return -1; + } + + bool written = false; + conn->on_write([&written] (bool) { written = true; }); // temp + + conn->write(data, len); + + // sometimes we can just write and forget + if (written) return len; + while (!written) { + OS::block(); + } + + conn->on_write(nullptr); // temp + return len; +} +ssize_t TCP_FD_Conn::recv(void* dest, size_t len, int) +{ + // if the connection is closed or closing: read returns 0 + if (conn->is_closed() || conn->is_closing()) return 0; + if (not conn->is_connected()) { + errno = ENOTCONN; + return -1; + } + // read some bytes from readq + int bytes = readq.read((char*) dest, len); + if (bytes) return bytes; + + bool done = false; + bytes = 0; + + // block and wait for more + conn->on_read(len, + [&done, &bytes, dest] (auto buffer) { + // copy the data itself + if (buffer->size() > 0) + memcpy(dest, buffer->data(), buffer->size()); + // we are done + done = true; + bytes = buffer->size(); + }); + + // BLOCK HERE + while (!done || !conn->is_readable()) { + OS::block(); + } + // restore + this->set_default_read(); + return bytes; +} +int TCP_FD_Conn::close() +{ + conn->close(); + // wait for connection to close completely + while (!conn->is_closed()) { + OS::block(); + } + return 0; +} +int TCP_FD_Conn::shutdown(int mode) +{ + if (not conn->is_connected()) { + errno = ENOTCONN; + return -1; + } + switch (mode) { + case SHUT_RDWR: + conn->close(); + return 0; + case SHUT_RD: + printf("Ignoring shutdown(SHUT_RD)\n"); + return 0; + case SHUT_WR: + printf("Ignoring shutdown(SHUT_WR)\n"); + return 0; + default: + errno = EINVAL; + return -1; + } +} + +/// socket as listener + +int TCP_FD_Listen::listen(int backlog) +{ + listener.on_connect( + [this, backlog] (net::tcp::Connection_ptr conn) + { + // remove oldest if full + if ((int) this->connq.size() >= backlog) + this->connq.pop_back(); + // new connection + this->connq.push_front(conn); + /// if someone is blocking they should be leaving right about now + }); + return 0; +} +int TCP_FD_Listen::accept(struct sockaddr *__restrict__ addr, socklen_t *__restrict__ addr_len) +{ + // block until connection appears + while (connq.empty()) { + OS::block(); + } + // retrieve connection from queue + auto sock = connq.back(); + connq.pop_back(); + // make sure socket is connected, as promised + if (not sock->is_connected()) { + errno = ENOTCONN; + return -1; + } + // create connected TCP socket + auto& fd = FD_map::_open(); + fd.cd = new TCP_FD_Conn(sock); + // set address and length + auto* sin = (sockaddr_in*) addr; + sin->sin_family = AF_INET; + sin->sin_port = sock->remote().port(); + sin->sin_addr.s_addr = sock->remote().address().whole; + *addr_len = sizeof(sockaddr_in); + // return socket + return fd.get_id(); +} +int TCP_FD_Listen::close() +{ + listener.close(); + return 0; +} diff --git a/src/posix/udp_fd.cpp b/src/posix/udp_fd.cpp new file mode 100644 index 0000000000..fff9c02a9b --- /dev/null +++ b/src/posix/udp_fd.cpp @@ -0,0 +1,387 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017-2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include // OS::block() +#include + +#define POSIX_STRACE +#ifdef POSIX_STRACE +#define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define PRINT(fmt, ...) /* fmt */ +#endif + +// return the "currently selected" networking stack +static net::Inet& net_stack() { + return net::Inet4::stack<> (); +} + +size_t UDP_FD::max_buffer_msgs() const +{ + return (rcvbuf_ / net_stack().udp().max_datagram_size()); +} + +void UDP_FD::recv_to_buffer(net::UDPSocket::addr_t addr, + net::UDPSocket::port_t port, const char* buf, size_t len) +{ + // only recv to buffer if not full + if(buffer_.size() < max_buffer_msgs()) { + // copy data into to-be Message buffer + auto buff = net::tcp::buffer_t(new std::vector (buf, buf + len)); + // emplace the message in buffer + buffer_.emplace_back(htonl(addr.whole), htons(port), std::move(buff)); + } +} + +int UDP_FD::read_from_buffer(void* buffer, size_t len, int flags, + struct sockaddr* address, socklen_t* address_len) +{ + assert(!buffer_.empty() && "Trying to read from empty buffer"); + + auto& msg = buffer_.front(); + auto& mbuf = msg.buffer; + + int bytes = std::min(len, mbuf->size()); + memcpy(buffer, mbuf->data(), bytes); + + if(address != nullptr) { + memcpy(address, &msg.src, std::min(*address_len, (uint32_t) sizeof(struct sockaddr_in))); + *address_len = sizeof(struct sockaddr_in); + } + + if(!(flags & MSG_PEEK)) + buffer_.pop_front(); + + return bytes; +} +void UDP_FD::set_default_recv() +{ + assert(this->sock != nullptr && "Default recv called on nullptr"); + this->sock->on_read({this, &UDP_FD::recv_to_buffer}); +} +UDP_FD::~UDP_FD() +{ + // shutdown underlying socket, makes sure no callbacks are called on dead fd + if(this->sock) + sock->close(); +} +ssize_t UDP_FD::read(void* buffer, size_t len) +{ + return recv(buffer, len, 0); +} +int UDP_FD::write(const void* buffer, size_t len) +{ + return send(buffer, len, 0); +} +int UDP_FD::close() +{ + return -1; +} +int UDP_FD::bind(const struct sockaddr* address, socklen_t len) +{ + // we can assume this has already been bound since there is a pointer + if(UNLIKELY(this->sock != nullptr)) { + errno = EINVAL; + return -1; + } + // invalid address + if(UNLIKELY(len != sizeof(struct sockaddr_in))) { + errno = EINVAL; + return -1; + } + // Bind + const auto port = ((sockaddr_in*)address)->sin_port; + auto& udp = net_stack().udp(); + try + { + this->sock = (port) ? &udp.bind(ntohs(port)) : &udp.bind(); + set_default_recv(); + PRINT("UDP: bind(%s)\n", sock->local().to_string().c_str()); + return 0; + } + catch(const net::UDP::Port_in_use_exception&) + { + errno = EADDRINUSE; + return -1; + } +} +int UDP_FD::connect(const struct sockaddr* address, socklen_t address_len) +{ + if (UNLIKELY(address_len < sizeof(struct sockaddr_in))) { + return -EINVAL; + } + + // If the socket has not already been bound to a local address, + // connect() shall bind it to an address which is an unused local address. + if(this->sock == nullptr) { + this->sock = &net_stack().udp().bind(); + set_default_recv(); + } + + const auto& addr = *((sockaddr_in*)address); + if (addr.sin_family == AF_UNSPEC) + { + peer_.sin_addr.s_addr = 0; + peer_.sin_port = 0; + } + else + { + peer_.sin_family = AF_INET; + peer_.sin_addr = addr.sin_addr; + peer_.sin_port = addr.sin_port; + } + PRINT("UDP: connect(%s:%u)\n", + net::IP4::addr(peer_.sin_addr.s_addr).to_string().c_str(), + htons(peer_.sin_port)); + + return 0; +} +ssize_t UDP_FD::send(const void* message, size_t len, int flags) +{ + if(!is_connected()) { + errno = EDESTADDRREQ; + return -1; + } + PRINT("UDP: send(%lu, %x)\n", len, flags); + + return sendto(message, len, flags, (struct sockaddr*)&peer_, sizeof(peer_)); +} + +ssize_t UDP_FD::sendto(const void* message, size_t len, int, + const struct sockaddr* dest_addr, socklen_t dest_len) +{ + // The specified address is not a valid address for the address family of the specified socket. + if(UNLIKELY(dest_len != sizeof(struct sockaddr_in))) { + errno = EINVAL; + return -1; + } + // Bind a socket if we dont already have one + if(this->sock == nullptr) { + this->sock = &net_stack().udp().bind(); + set_default_recv(); + } + const auto& dest = *((sockaddr_in*)dest_addr); + // If the socket protocol supports broadcast and the specified address + // is a broadcast address for the socket protocol, + // sendto() shall fail if the SO_BROADCAST option is not set for the socket. + if(!broadcast_ && dest.sin_addr.s_addr == INADDR_BROADCAST) { // Fix me + return -1; + } + + // Sending + bool written = false; + this->sock->sendto(ntohl(dest.sin_addr.s_addr), ntohs(dest.sin_port), message, len, + [&written]() { written = true; }); + + while(!written) + OS::block(); + + return len; +} +ssize_t UDP_FD::recv(void* buffer, size_t len, int flags) +{ + PRINT("UDP: recv(%lu, %x)\n", len, flags); + return recvfrom(buffer, len, flags, nullptr, 0); +} +ssize_t UDP_FD::recvfrom(void *__restrict__ buffer, size_t len, int flags, + struct sockaddr *__restrict__ address, socklen_t *__restrict__ address_len) +{ + if(UNLIKELY(this->sock == nullptr)) { + errno = EINVAL; // + return -1; + } + + // Read from buffer if not empty + if(!buffer_.empty()) + { + return read_from_buffer(buffer, len, flags, address, address_len); + } + // Else make a blocking receive + else + { + int bytes = 0; + bool done = false; + + this->sock->on_read(net::UDPSocket::recvfrom_handler::make_packed( + [&bytes, &done, this, + buffer, len, flags, address, address_len] + (net::UDPSocket::addr_t addr, net::UDPSocket::port_t port, + const char* data, size_t data_len) + { + // if this already been called once while blocking, buffer + if(done) { + recv_to_buffer(addr, port, data, data_len); + return; + } + + bytes = std::min(len, data_len); + memcpy(buffer, data, bytes); + + // TODO: If the actual length of the address is greater than the length of the supplied sockaddr structure, + // the stored address shall be truncated. + if(address != nullptr) { + auto& sender = *((sockaddr_in*)address); + sender.sin_family = AF_INET; + sender.sin_port = htons(port); + sender.sin_addr.s_addr = htonl(addr.whole); + *address_len = sizeof(struct sockaddr_in); + } + done = true; + + // Store in buffer if PEEK + if(flags & MSG_PEEK) + recv_to_buffer(addr, port, data, data_len); + })); + + // Block until (any) data is read + while(!done) + OS::block(); + + set_default_recv(); + + return bytes; + } +} +int UDP_FD::getsockopt(int level, int option_name, + void *option_value, socklen_t *option_len) +{ + PRINT("UDP: getsockopt(%d, %d)\n", level, option_name); + if(level != SOL_SOCKET) + return -1; + + switch(option_name) + { + case SO_ACCEPTCONN: + { + errno = ENOPROTOOPT; + return -1; + } + case SO_BROADCAST: + { + if(*option_len < (int)sizeof(int)) + { + errno = EINVAL; + return -1; + } + + *((int*)option_value) = broadcast_; + *option_len = sizeof(broadcast_); + return 0; + } + case SO_KEEPALIVE: + { + errno = ENOPROTOOPT; + return -1; + } + case SO_RCVBUF: + { + if(*option_len < (int)sizeof(int)) + { + errno = EINVAL; + return -1; + } + + *((int*)option_value) = rcvbuf_; + *option_len = sizeof(rcvbuf_); + return 0; + } + // Address can always be reused in IncludeOS + case SO_REUSEADDR: + { + if(*option_len < (int)sizeof(int)) + { + errno = EINVAL; + return -1; + } + + *((int*)option_value) = 1; + *option_len = sizeof(int); + return 0; + } + case SO_TYPE: + { + if(*option_len < (int)sizeof(int)) + { + errno = EINVAL; + return -1; + } + + *((int*)option_value) = SOCK_DGRAM; + *option_len = sizeof(int); + return 0; + } + + default: + return -1; + } // < switch(option_name) +} + +int UDP_FD::setsockopt(int level, int option_name, + const void *option_value, socklen_t option_len) +{ + PRINT("UDP: setsockopt(%d, %d, ... %d)\n", level, option_name, option_len); + if(level != SOL_SOCKET) + return -1; + + switch(option_name) + { + case SO_BROADCAST: + { + if(option_len < (int)sizeof(int)) + return -1; + + broadcast_ = *((int*)option_value); + return 0; + } + case SO_KEEPALIVE: + { + errno = ENOPROTOOPT; + return -1; + } + case SO_RCVBUF: + { + if(option_len < (int)sizeof(int)) + return -1; + + rcvbuf_ = *((int*)option_value); + return 0; + } + // Address can always be reused in IncludeOS + case SO_REUSEADDR: + { + return 0; + } + + default: + return -1; + } // < switch(option_name) +} + +/// socket default handler getters + +UDP_FD::on_read_func UDP_FD::get_default_read_func() +{ + return [] (net::tcp::buffer_t) {}; +} +UDP_FD::on_write_func UDP_FD::get_default_write_func() +{ + return [] {}; +} +UDP_FD::on_except_func UDP_FD::get_default_except_func() +{ + return [] {}; +} From 326cedae51f1f7d8a93b955ef2b7452bf5f79d8f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 9 Apr 2018 13:20:53 +0200 Subject: [PATCH 140/723] openssl: Use new updated bundle from 1.0.2 stable --- cmake/openssl.cmake | 10 ++++------ cmake/post.service.cmake | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/cmake/openssl.cmake b/cmake/openssl.cmake index 53fdb92c90..7556812f7e 100644 --- a/cmake/openssl.cmake +++ b/cmake/openssl.cmake @@ -2,11 +2,11 @@ include(ExternalProject) if(${ARCH} STREQUAL "x86_64") - set(OPENSSL_HASH 8f807887c7f9835a7fc4a54b0d3cce55) + set(OPENSSL_HASH 3ef6bc4e8be049725ca3887f0d235031) ExternalProject_Add(openssl_bundle PREFIX openssl - URL https://github.com/fwsGonzo/OpenSSL_bundle/releases/download/v1.0/openssl_bundle.tar.gz + URL https://github.com/fwsGonzo/OpenSSL_bundle/releases/download/v1.2/openssl_bundle.tar.gz URL_HASH MD5=${OPENSSL_HASH} CONFIGURE_COMMAND "" BUILD_COMMAND "" @@ -16,9 +16,8 @@ if(${ARCH} STREQUAL "x86_64") set(OPENSSL_DIR ${CMAKE_CURRENT_BINARY_DIR}/openssl/src/openssl_bundle) set(OPENSSL_INCLUDE ${OPENSSL_DIR}/include) - set(OPENSSL_CONF ${OPENSSL_DIR}/openssl/opensslconf.h) - set(OPENSSL_LIB_CRYPTO ${OPENSSL_DIR}/libcrypto.a) - set(OPENSSL_LIB_SSL ${OPENSSL_DIR}/libssl.a) + set(OPENSSL_LIB_CRYPTO ${OPENSSL_DIR}/lib/libcrypto.a) + set(OPENSSL_LIB_SSL ${OPENSSL_DIR}/lib/libssl.a) add_library(openssl_ssl STATIC IMPORTED) set_target_properties(openssl_ssl PROPERTIES IMPORTED_LOCATION ${OPENSSL_LIB_SSL}) @@ -28,5 +27,4 @@ if(${ARCH} STREQUAL "x86_64") install(FILES ${OPENSSL_LIB_CRYPTO} DESTINATION includeos/${ARCH}/lib) install(FILES ${OPENSSL_LIB_SSL} DESTINATION includeos/${ARCH}/lib) install(DIRECTORY ${OPENSSL_INCLUDE} DESTINATION includeos/${ARCH}/include) - install(FILES ${OPENSSL_CONF} DESTINATION includeos/${ARCH}/include/openssl) endif() diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 413d4e696c..e5300bbfe9 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -306,7 +306,7 @@ if(${ARCH} STREQUAL "x86_64") set_target_properties(libcrypto PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libcrypto.a) set(OPENSSL_LIBS libssl libcrypto) - include_directories(${INSTALL_LOC}/${ARCH}/include/include) + include_directories(${INSTALL_LOC}/${ARCH}/include) endif() add_library(libosdeps STATIC IMPORTED) From 77b2ae8a1c241a0cef579bedc49a897445411cd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 9 Apr 2018 14:37:58 +0200 Subject: [PATCH 141/723] musl: UDP socket now working correctly --- api/posix/udp_fd.hpp | 1 - src/net/ip4/arp.cpp | 6 ++--- src/posix/udp_fd.cpp | 32 +++++++++++--------------- test/posix/integration/udp/service.cpp | 4 +++- test/posix/integration/udp/test.py | 30 ++++++++++++++++++------ 5 files changed, 43 insertions(+), 30 deletions(-) diff --git a/api/posix/udp_fd.hpp b/api/posix/udp_fd.hpp index 8e9a0dd169..5923ea8d6e 100644 --- a/api/posix/udp_fd.hpp +++ b/api/posix/udp_fd.hpp @@ -48,7 +48,6 @@ class UDP_FD : public SockFD { int bind(const struct sockaddr *, socklen_t) override; int connect(const struct sockaddr *, socklen_t) override; - ssize_t send(const void *, size_t, int fl) override; ssize_t sendto(const void *, size_t, int, const struct sockaddr *, socklen_t) override; ssize_t recv(void*, size_t, int fl) override; diff --git a/src/net/ip4/arp.cpp b/src/net/ip4/arp.cpp index a83eafa6b3..38daecafcd 100644 --- a/src/net/ip4/arp.cpp +++ b/src/net/ip4/arp.cpp @@ -89,7 +89,7 @@ namespace net { case H_reply: { // Stat increment replies received replies_rx_++; - PRINT("\t ARP REPLY: %s belongs to %s (waiting: %u)\n", + PRINT("\t ARP REPLY: %s belongs to %s (waiting: %zu)\n", hdr->sipaddr.str().c_str(), hdr->shwaddr.str().c_str(), waiting_packets_.size()); break; } @@ -106,7 +106,7 @@ namespace net { auto entry = cache_.find(ip); if (entry != cache_.end()) { - PRINT("Cached entry found: %s recorded @ %llu. Updating timestamp\n", + PRINT("Cached entry found: %s recorded @ %zu. Updating timestamp\n", entry->second.mac().str().c_str(), entry->second.timestamp()); if (entry->second.mac() != mac) { @@ -142,7 +142,7 @@ namespace net { res->source_ip().str().c_str(), res->source_mac().str().c_str()); MAC::Addr dest = hdr_in->shwaddr; - PRINT(" physical> Sending response to %s. Linklayer begin: buf + %i \n", + PRINT(" physical> Sending response to %s. Linklayer begin: buf + %li \n", dest.str().c_str(), res->layer_begin() - res->buf() ); linklayer_out_(std::move(res), dest, Ethertype::ARP); diff --git a/src/posix/udp_fd.cpp b/src/posix/udp_fd.cpp index fff9c02a9b..086d6275b3 100644 --- a/src/posix/udp_fd.cpp +++ b/src/posix/udp_fd.cpp @@ -18,7 +18,7 @@ #include // OS::block() #include -#define POSIX_STRACE +//#define POSIX_STRACE 1 #ifdef POSIX_STRACE #define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else @@ -89,7 +89,7 @@ int UDP_FD::write(const void* buffer, size_t len) } int UDP_FD::close() { - return -1; + return 0; } int UDP_FD::bind(const struct sockaddr* address, socklen_t len) { @@ -150,36 +150,32 @@ int UDP_FD::connect(const struct sockaddr* address, socklen_t address_len) return 0; } -ssize_t UDP_FD::send(const void* message, size_t len, int flags) -{ - if(!is_connected()) { - errno = EDESTADDRREQ; - return -1; - } - PRINT("UDP: send(%lu, %x)\n", len, flags); - - return sendto(message, len, flags, (struct sockaddr*)&peer_, sizeof(peer_)); -} ssize_t UDP_FD::sendto(const void* message, size_t len, int, const struct sockaddr* dest_addr, socklen_t dest_len) { - // The specified address is not a valid address for the address family of the specified socket. - if(UNLIKELY(dest_len != sizeof(struct sockaddr_in))) { - errno = EINVAL; - return -1; + if(not is_connected()) + { + if(UNLIKELY((dest_addr == nullptr or dest_len == 0))) { + return -EDESTADDRREQ; + } + // The specified address is not a valid address for the address family of the specified socket. + else if(UNLIKELY(dest_len != sizeof(struct sockaddr_in))) { + return -EAFNOSUPPORT; + } } + // Bind a socket if we dont already have one if(this->sock == nullptr) { this->sock = &net_stack().udp().bind(); set_default_recv(); } - const auto& dest = *((sockaddr_in*)dest_addr); + const auto& dest = (not is_connected()) ? *((sockaddr_in*)dest_addr) : peer_; // If the socket protocol supports broadcast and the specified address // is a broadcast address for the socket protocol, // sendto() shall fail if the SO_BROADCAST option is not set for the socket. if(!broadcast_ && dest.sin_addr.s_addr == INADDR_BROADCAST) { // Fix me - return -1; + return -EOPNOTSUPP; } // Sending diff --git a/test/posix/integration/udp/service.cpp b/test/posix/integration/udp/service.cpp index e5eb2e25e1..a32813bfc6 100644 --- a/test/posix/integration/udp/service.cpp +++ b/test/posix/integration/udp/service.cpp @@ -97,11 +97,12 @@ int main() destaddr.sin_port = htons(OUT_PORT); const char *my_message = "Only hipsters uses POSIX"; - res = sendto(socket(AF_INET, SOCK_DGRAM, 0), my_message, strlen(my_message), 0, (struct sockaddr *)&destaddr, sizeof(destaddr)); CHECKSERT(res > 0, "Message was sent from NEW socket to %s:%u (verified by script)", inet.gateway().to_string().c_str(), OUT_PORT); + INFO2("sendto() called"); + INFO("UDP Socket", "send() and connect()"); int fd_send_connect = socket(AF_INET, SOCK_DGRAM, 0); @@ -115,6 +116,7 @@ int main() res = send(fd_send_connect, my_message, strlen(my_message), 0); CHECKSERT(res > 0, "Send works when connected (verified by script)"); + INFO2("sendto() called"); INFO("UDP Socket", "reading from buffer with recv()"); int i = 0; diff --git a/test/posix/integration/udp/test.py b/test/posix/integration/udp/test.py index 533c49aab2..dc019783c7 100755 --- a/test/posix/integration/udp/test.py +++ b/test/posix/integration/udp/test.py @@ -16,12 +16,19 @@ import socket # Set up a temporary interface -subprocess.call(["sudo", "ifconfig", "bridge43:1", "10.0.0.3/24"]) +import platform +if platform.system() == 'Darwin': + subprocess.call(["sudo", "ifconfig", "bridge43", "alias", "10.0.0.3/24"]) +else: + subprocess.call(["sudo", "ifconfig", "bridge43:1", "10.0.0.3/24"]) # Tear down interface on exit @atexit.register def tear_down(): - subprocess.call(["sudo", "ifconfig", "bridge43:1", "down"]) + if platform.system() == 'Darwin': + subprocess.call(["sudo", "ifconfig", "bridge43", "-alias", "10.0.0.3"]) + else: + subprocess.call(["sudo", "ifconfig", "bridge43:1", "down"]) S_HOST, S_PORT = '10.0.0.3', 4242 @@ -34,6 +41,8 @@ def tear_down(): HOST, PORT = '10.0.0.50', 1042 +RECEIVED = '' + def UDP_send(trigger_line): MESSAGE = "POSIX is for hipsters" sock = socket.socket @@ -41,9 +50,13 @@ def UDP_send(trigger_line): sock.bind((S_HOST, S_PORT + 1)) sock.sendto(MESSAGE, (HOST, PORT)) -def UDP_recv(trigger_line): - received = server.recv(1024) - return received == S_MESSAGE +def UDP_recv(): + RECEIVED = server.recv(1024) + +def verify_recv(trigger_line): + ok = RECEIVED == S_MESSAGE + RECEIVED = '' + return ok def UDP_send_much(trigger_line): MESSAGE = "Message #" @@ -56,10 +69,13 @@ def UDP_send_much(trigger_line): sock.send(MESSAGE + `i`) print "Sending", MESSAGE + `i` +import thread +thread.start_new_thread(UDP_recv, ()) + # Add custom event-handler vm.on_output("recvfrom()", UDP_send) -vm.on_output("sendto()", UDP_recv) -vm.on_output("send() and connect()", UDP_recv) +vm.on_output("sendto() called", verify_recv) +vm.on_output("sendto() called", verify_recv) vm.on_output("reading from buffer", UDP_send_much) # Boot the VM, taking a timeout as parameter From e4b3f8d5ee674960c0ddde1ee2a4c9b0f525a1b1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 10 Apr 2018 10:11:48 +0200 Subject: [PATCH 142/723] Various OpenSSL related work --- api/net/openssl/tls_stream.hpp | 17 ++++++++++++++ examples/http_client/CMakeLists.txt | 4 +++- examples/http_client/nacl.txt | 9 ++------ examples/http_client/service.cpp | 21 +++++++++++------ examples/http_client/vm.json | 2 +- src/net/openssl/client.cpp | 35 ++++++++++++++++++++++++++--- src/net/openssl/init.cpp | 5 +++++ vmrunner/vm.schema.json | 2 +- 8 files changed, 75 insertions(+), 20 deletions(-) diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index 33c93c1943..c745e20df6 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -4,6 +4,13 @@ #include #include +#define VERBOSE_OPENSSL +#ifdef VERBOSE_OPENSSL +#define TLS_PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define TLS_PRINT(fmt, ...) /* fmt */ +#endif + namespace openssl { struct TLS_stream : public net::Stream @@ -174,6 +181,12 @@ namespace openssl } else if (status == STATUS_FAIL) { + if (num < 0) { + TLS_PRINT("TLS_stream::SSL_do_handshake() returned %d\n", num); + #ifdef VERBOSE_OPENSSL + ERR_print_errors_fp(stdout); + #endif + } this->close(); return; } @@ -247,10 +260,14 @@ namespace openssl { do { n = tls_perform_stream_write(); + if (n < 0) { + TLS_PRINT("TLS_stream::tls_perform_handshake() stream write failed\n"); + } } while (n > 0); return n; } else { + TLS_PRINT("TLS_stream::tls_perform_handshake() returned %d\n", ret); this->close(); return -1; } diff --git a/examples/http_client/CMakeLists.txt b/examples/http_client/CMakeLists.txt index e3b9233a8f..6fbfcc8118 100644 --- a/examples/http_client/CMakeLists.txt +++ b/examples/http_client/CMakeLists.txt @@ -25,6 +25,8 @@ set(DRIVERS boot_logger ) +set(PLUGINS vfs) + include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) -diskbuilder(certs) +diskbuilder(disk) diff --git a/examples/http_client/nacl.txt b/examples/http_client/nacl.txt index 75f41f7e04..890292e66b 100644 --- a/examples/http_client/nacl.txt +++ b/examples/http_client/nacl.txt @@ -1,7 +1,2 @@ -Iface eth0 { - index: 0, - address: 10.0.0.42, - netmask: 255.255.255.0, - gateway: 10.0.0.1, - dns: 8.8.8.8 -} +Iface eth0 dhcp +eth0.index: 0 diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index 647e593a9f..9b4c92c435 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -25,11 +25,11 @@ static SSL_CTX* init_ssl_context() assert(!err); }); - auto ents = disk.fs().ls("/"); + auto ents = disk.fs().ls("/mozilla"); // initialize client context openssl::init(); - return openssl::create_client(ents); + return openssl::create_client(ents, true); } #include @@ -37,12 +37,10 @@ static SSL_CTX* init_ssl_context() #include #include -void Service::start() +static void begin_http(net::Inet& inet) { - auto& inet = net::Super_stack::get(0); - using namespace http; - + /* static Basic_client basic{inet.tcp()}; const std::string url{"http://www.google.com"}; @@ -58,7 +56,7 @@ void Service::start() printf("Make sure the virtual machine can reach internet.\n"); } }); - + */ auto* ctx = init_ssl_context(); assert(ctx != nullptr); @@ -78,3 +76,12 @@ void Service::start() } }); } + +void Service::start() +{ + auto& inet = net::Super_stack::get(0); + inet.on_config( + [] (auto& inet) { + begin_http(inet); + }); +} diff --git a/examples/http_client/vm.json b/examples/http_client/vm.json index d0dae980d0..17b60814b0 100644 --- a/examples/http_client/vm.json +++ b/examples/http_client/vm.json @@ -1,5 +1,5 @@ { "net" : [ - {"device" : "virtio", "mac" : "c0:01:0a:00:00:2a"} + {"device" : "virtio"} ] } diff --git a/src/net/openssl/client.cpp b/src/net/openssl/client.cpp index 1592d0d976..d90d9e81ac 100644 --- a/src/net/openssl/client.cpp +++ b/src/net/openssl/client.cpp @@ -58,21 +58,29 @@ tls_init_client(fs::List ents) int res = SSL_CTX_set_cipher_list(ctx, "AES256-SHA"); assert(res == 1); + /* X509_STORE* store = X509_STORE_new(); assert(store != nullptr); + int certs = 0; for (auto& ent : ents) { if (ent.is_file()) { - INFO2("Loading certificate %s", ent.name().c_str()); + TLS_PRINT("\t\t* %s\n", ent.name().c_str()); auto buffer = ent.read(0, ent.size()); tls_load_from_memory(store, buffer); + certs++; } } + INFO2("Loaded %d certificates", certs); // assign CA store to CTX SSL_CTX_set_cert_store(ctx, store); + */ + (void) ents; + + SSL_CTX_load_verify_locations(ctx, nullptr, "/certs"); // create private key for client tls_private_key_for_ctx(ctx); @@ -84,6 +92,27 @@ tls_init_client(fs::List ents) return ctx; } +extern "C" typedef int verify_cb_t(int, X509_STORE_CTX*); +static verify_cb_t verify_cb; +int verify_cb(int preverify, X509_STORE_CTX* ctx) +{ + printf("verify_cb\n"); + X509* current_cert = X509_STORE_CTX_get_current_cert(ctx); + + if (preverify == 0) + { + if (current_cert) { + X509_NAME_print_ex_fp(stdout, + X509_get_subject_name(current_cert), + 0, XN_FLAG_ONELINE); + printf("\n"); + } + TLS_PRINT("verify_cb error: %d\n", X509_STORE_CTX_get_error(ctx)); + ERR_print_errors_fp(stdout); + } + return preverify; +} + namespace openssl { SSL_CTX* create_client(fs::List ents, bool verify_peer) @@ -97,8 +126,8 @@ namespace openssl void client_verify_peer(SSL_CTX* ctx) { - SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, nullptr); - SSL_CTX_set_verify_depth(ctx, 1); + SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER|SSL_VERIFY_CLIENT_ONCE, &verify_cb); + SSL_CTX_set_verify_depth(ctx, 20); tls_check_for_errors(); } } diff --git a/src/net/openssl/init.cpp b/src/net/openssl/init.cpp index 58ece7007e..c91128f6d2 100644 --- a/src/net/openssl/init.cpp +++ b/src/net/openssl/init.cpp @@ -66,10 +66,15 @@ namespace openssl { INFO("OpenSSL", "Initializing (%s)", OPENSSL_VERSION_TEXT); init_once = true; + printf("setup_rng\n"); setup_rng(); + printf("SSL_library_init\n"); SSL_library_init(); // OPENSSL_INIT_LOAD_SSL_STRINGS | OPENSSL_INIT_LOAD_CRYPTO_STRINGS + printf("SSL_load_error_strings\n"); SSL_load_error_strings(); + printf("ERR_load_crypto_strings\n"); ERR_load_crypto_strings(); + printf("ERR_load_BIO_strings\n"); ERR_load_BIO_strings(); } } diff --git a/vmrunner/vm.schema.json b/vmrunner/vm.schema.json index 705317567d..9b5fd91f01 100644 --- a/vmrunner/vm.schema.json +++ b/vmrunner/vm.schema.json @@ -75,7 +75,7 @@ "properties" : { "device" : { "type" : "string" }, "name" : { "type" : "string" }, - "backend" : { "enum" : ["tap", "user"], "default" : "tap" } + "backend" : { "enum" : ["tap", "user"], "default" : "user" } }, "required" : ["device"] From 1473b819692ae51505a0cd10507b971f7a781b6f Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 10 Apr 2018 11:04:24 +0200 Subject: [PATCH 143/723] musl: new bundle making socketcall use __syscall_ret --- cmake/cross_compiled_libraries.cmake | 2 +- etc/build_musl.sh | 10 +++------- etc/musl/syscall.h | 5 ++--- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/cmake/cross_compiled_libraries.cmake b/cmake/cross_compiled_libraries.cmake index 256f3237d9..74cb21c669 100644 --- a/cmake/cross_compiled_libraries.cmake +++ b/cmake/cross_compiled_libraries.cmake @@ -19,7 +19,7 @@ else(BUNDLE_LOC) ExternalProject_Add(PrecompiledLibraries PREFIX precompiled URL https://github.com/hioa-cs/IncludeOS/releases/download/v0.12.0-rc.2/IncludeOS_dependencies_v0-12-0_musl_libunwind_singlethreaded.tar.gz - URL_HASH SHA1=febb731b35130431871d46476ce673781e9ce8db + URL_HASH SHA1=d011b393fff5eba6df865ffb085628a105e9404d CONFIGURE_COMMAND "" BUILD_COMMAND "" UPDATE_COMMAND "" diff --git a/etc/build_musl.sh b/etc/build_musl.sh index c95f242fce..8209f4da15 100755 --- a/etc/build_musl.sh +++ b/etc/build_musl.sh @@ -12,6 +12,9 @@ fi pushd musl +git checkout $musl_version +make distclean || true + # Replace syscall API cp $INCLUDEOS_SRC/api/syscalls.h src/internal/includeos_syscalls.h cp $INCLUDEOS_SRC/etc/musl/syscall.h src/internal/ @@ -23,13 +26,6 @@ git apply $INCLUDEOS_SRC/etc/musl/musl.patch || true git apply $INCLUDEOS_SRC/etc/musl/endian.patch || true -# ln -s $INCLUDEOS_SRC/api/arch/syscalls.h arch/x86_64/syscall_arch.h -# ln -s $INCLUDEOS_SRC/api/arch/syscalls.h arch/i386/syscall_arch.h - - -git checkout $musl_version -make distclean || true - export CFLAGS="$CFLAGS -g -target $ARCH-pc-linux-elf" ./configure --prefix=$TEMP_INSTALL_DIR/$TARGET --disable-shared --enable-debug --target=$TARGET #--enable-optimize=* make $num_jobs diff --git a/etc/musl/syscall.h b/etc/musl/syscall.h index 705d8c352a..0846fc0254 100644 --- a/etc/musl/syscall.h +++ b/etc/musl/syscall.h @@ -42,13 +42,12 @@ long __syscall_ret(unsigned long), __syscall(syscall_arg_t, ...), #define socketcall __socketcall -#define socketcall_cp __socketcall_cp +#define socketcall_cp __socketcall #define __syscall_cp syscall #define syscall_cp syscall -#define __socketcall(nm, ...) socketcall_##nm(__VA_ARGS__) -#define __socketcall_cp(nm, ...) __syscall_ret(socketcall_##nm(__VA_ARGS__)) +#define __socketcall(nm, ...) __syscall_ret(socketcall_##nm(__VA_ARGS__)) /* fixup legacy 16-bit junk */ From 44ca7b1f0bf9358d5aadc06d8ab4d90a58c83a17 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 10 Apr 2018 11:25:49 +0200 Subject: [PATCH 144/723] cmake: make lest option depend on test option --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2e11674674..d15a65969a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,15 +201,15 @@ if(examples) add_subdirectory(examples) endif(examples) +option(tests "Build unit tests in /test and install lest test framework" OFF) option(lest "Install lest unittest headers" OFF) -if (lest) + +if (lest OR tests) init_submodule(test/lest) install(DIRECTORY test/lest/include/lest DESTINATION ${CMAKE_INSTALL_PREFIX}/includeos/include) endif() -option(tests "Build unit tests in /test and install lest test framework" OFF) if(tests) - set(lest ON) enable_testing() ExternalProject_Add(unittests PREFIX unittests From 18ea4dbfa22140620e836d5b261d0ec258ca79c1 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 10 Apr 2018 11:45:21 +0200 Subject: [PATCH 145/723] unit: update unit tests to musl/non-throwing fd --- test/posix/unit/fd_map_test.cpp | 7 +++---- test/posix/unit/inet_test.cpp | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/test/posix/unit/fd_map_test.cpp b/test/posix/unit/fd_map_test.cpp index 610a987694..c7fd0216ab 100644 --- a/test/posix/unit/fd_map_test.cpp +++ b/test/posix/unit/fd_map_test.cpp @@ -64,9 +64,8 @@ CASE("Adding a implemented FD descriptor in FD_map") // Close works FD_map::close(id); - // Throws when not found works - EXPECT_THROWS_AS(FD_map::_get(id), FD_not_found); + EXPECT(FD_map::_get(id) == nullptr); - const int bet = 322; // this is a throw - EXPECT_THROWS_AS(FD_map::_get(bet), FD_not_found); + const int bet = 322; // this used to be a throw + EXPECT(FD_map::_get(bet) == nullptr); } diff --git a/test/posix/unit/inet_test.cpp b/test/posix/unit/inet_test.cpp index 109f2a2f9f..7ef24df778 100644 --- a/test/posix/unit/inet_test.cpp +++ b/test/posix/unit/inet_test.cpp @@ -60,7 +60,7 @@ CASE("IPv4 address manipulation - addr to pointer") // Too small address buffer yeilds error char addrstr2[11]; res = inet_ntop(AF_INET, &addr, (char*)&addrstr2, sizeof(addrstr2)); - EXPECT(errno == ENOSPC); + EXPECT(errno == EAFNOSUPPORT); // No buffer returns nullptr res = inet_ntop(AF_INET, &addr, nullptr, 0); @@ -85,7 +85,7 @@ CASE("IPv4 address manipulation - pointer to addr") res = inet_pton(AF_INET, "256.666.666.1338", &addr); EXPECT(res == 0); - // Unsupported IP4 format + // IP4 format invalid res = inet_pton(AF_INET, "255.255.0", &addr); - EXPECT(res == 1); + EXPECT(res == 0); } From 2ded015c5fe53fcbb79c27c0475531f14c8fede7 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 10 Apr 2018 12:51:18 +0200 Subject: [PATCH 146/723] test: make modules test use x86_nano for 32-bit --- test/kernel/integration/modules/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test/kernel/integration/modules/CMakeLists.txt b/test/kernel/integration/modules/CMakeLists.txt index 06997e26ca..9613d4d5a6 100644 --- a/test/kernel/integration/modules/CMakeLists.txt +++ b/test/kernel/integration/modules/CMakeLists.txt @@ -5,6 +5,7 @@ if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) endif() set(ARCH i686) +set(PLATFORM x86_nano) include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) From ccd4b6a65eb498351ba2922d63f8207fc0b50797 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 10 Apr 2018 13:32:13 +0200 Subject: [PATCH 147/723] test: Don't check mode of elements in .tar.gz test --- test/util/integration/tar_gz/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/util/integration/tar_gz/test.py b/test/util/integration/tar_gz/test.py index c3f852c5b2..7f645c3a78 100755 --- a/test/util/integration/tar_gz/test.py +++ b/test/util/integration/tar_gz/test.py @@ -58,7 +58,7 @@ def check_num_outputs(line): # README.md's header vm.on_output("tar_example/l1_f1/l2/README.md - name of README.md", increment_header1) -vm.on_output("Mode of README.md: 0000664", increment_header1) +vm.on_output("Mode of README.md: 0000", increment_header1) vm.on_output("Uid of README.md: ", increment_header1) vm.on_output("Gid of README.md: ", increment_header1) vm.on_output("Size of README.md: 293", increment_header1) @@ -83,7 +83,7 @@ def check_num_outputs(line): # Folder l2's header and content (no content) vm.on_output("tar_example/l1_f1/l2/ - name of l2", increment_header2) -vm.on_output("Mode of l2: 0000775", increment_header2) +vm.on_output("Mode of l2: 0000", increment_header2) vm.on_output("Uid of l2: ", increment_header2) vm.on_output("Gid of l2: ", increment_header2) vm.on_output("Size of l2: 0", increment_header2) From 1e376aac3e39bf384bbc78ff94a68de5cb11c096 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 10 Apr 2018 14:30:36 +0200 Subject: [PATCH 148/723] test: make exceptions test verify panic on uncaught exception --- test/stl/integration/exceptions/service.cpp | 4 ++- test/stl/integration/exceptions/test.py | 29 ++++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/test/stl/integration/exceptions/service.cpp b/test/stl/integration/exceptions/service.cpp index 0a8dcda2e2..e4e0aaf331 100644 --- a/test/stl/integration/exceptions/service.cpp +++ b/test/stl/integration/exceptions/service.cpp @@ -69,5 +69,7 @@ void Service::start(const std::string&) MYINFO ("Running LEST-tests"); auto failed = lest::run(tests, {"-p"}); assert(not failed); - MYINFO("SUCCESS"); + + MYINFO("Part 1 OK. Now throwing whithout try-catch which should panic"); + throw std::runtime_error("Uncaught exception expecting panic"); } diff --git a/test/stl/integration/exceptions/test.py b/test/stl/integration/exceptions/test.py index f4f19408b6..59cd0dc3bd 100755 --- a/test/stl/integration/exceptions/test.py +++ b/test/stl/integration/exceptions/test.py @@ -8,4 +8,31 @@ sys.path.insert(0,includeos_src) from vmrunner import vmrunner -vmrunner.vms[0].cmake().boot(40).clean() + +vm = vmrunner.vms[0] + +tests_ok = 0 + +def test_ok(line): + global tests_ok + tests_ok += 1 + if (tests_ok == 2): + vm.exit(0, "All tests passed") + +def expected_panic(line): + print " VM panicked" + if (tests_ok == 1): + return True + else: + return False + +def test_fail(line): + print "Test didn't get expected panic output before end of backtrace" + return False + +vm.on_output("Part 1 OK", test_ok) +vm.on_panic(expected_panic, False) +vm.on_output("Uncaught exception expecting panic", test_ok) +vm.on_output("long_mode", test_fail) + +vm.cmake().boot(30).clean() From 0896b84466060acbb8be1d5da8b50838cb2f7a06 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 10 Apr 2018 14:33:12 +0200 Subject: [PATCH 149/723] kernel: add back abort_message, hack to override musl abort --- src/kernel/syscalls.cpp | 11 +++++++++++ src/service_name.cpp | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index 207bd140d8..69391a2680 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -38,6 +38,17 @@ static const char* panic_signature = "\x15\x07\t**** PANIC ****"; extern uintptr_t heap_begin; extern uintptr_t heap_end; +extern "C" __attribute__((noreturn)) +void abort_message(const char* format, ...) +{ + static char abort_buf[2048]; + va_list list; + va_start(list, format); + vsnprintf(abort_buf, sizeof(abort_buf), format, list); + va_end(list); + panic(abort_buf); +} + void _exit(int status) { SYSINFO("Service exiting with status %d", status); default_exit(); diff --git a/src/service_name.cpp b/src/service_name.cpp index e21d0d9d3a..a21e21a938 100644 --- a/src/service_name.cpp +++ b/src/service_name.cpp @@ -17,5 +17,13 @@ #include +extern "C" __attribute__((noreturn)) +void panic(const char* reason); + +__attribute__((noreturn)) +extern "C" void abort(){ + panic("Abort called"); +} + const char* service_name__ = SERVICE_NAME; const char* service_binary_name__ = SERVICE; From 6ed4a77f643ad6b74d8897804a2d5c4c5032cf6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 10 Apr 2018 14:54:18 +0200 Subject: [PATCH 150/723] util: tar decompress now returns a blob to avoid temp memory being used in tar files --- api/util/tar.hpp | 86 +++++++++--------------- lib/mender/include/mender/artifact.hpp | 8 +-- lib/mender/src/artifact.cpp | 29 ++++---- lib/mender/src/client.cpp | 4 +- src/util/tar.cpp | 50 +++++++++++++- test/util/integration/tar_gz/service.cpp | 4 +- test/util/integration/tar_gz/test.py | 4 +- 7 files changed, 100 insertions(+), 85 deletions(-) diff --git a/api/util/tar.hpp b/api/util/tar.hpp index bddb20b2ba..6a7adf48a6 100644 --- a/api/util/tar.hpp +++ b/api/util/tar.hpp @@ -78,6 +78,8 @@ static const int DECOMPRESSION_SIZE = 256; static bool has_uzlib_init = false; +using Tar_data = std::vector; + // --------------------------- Tar_exception --------------------------- class Tar_exception : public std::runtime_error { @@ -148,13 +150,20 @@ class Element { class Tar { public: + using Elements = std::vector; Tar() = default; int num_elements() const noexcept { return elements_.size(); } void add_element(const Element& element) { elements_.push_back(element); } const Element& element(const std::string& path) const; - const std::vector& elements() const noexcept { return elements_; } + const Elements& elements() const noexcept { return elements_; } std::vector element_names() const; + auto begin() const noexcept + { return elements_.begin(); } + + auto end() const noexcept + { return elements_.end(); } + private: std::vector elements_; @@ -163,76 +172,41 @@ class Tar { // ------------------------------ Reader ------------------------------ class Reader { - public: - uint32_t checksum(Element& element) const { return crc32(element.content(), element.size()); } - - unsigned int decompressed_length(const uint8_t* data, size_t size) const; - - Tar read_binary_tar() { + static Tar read_tar() + { const uint8_t* bin_content = &_binary_input_bin_start; const int bin_size = (intptr_t) &_binary_input_bin_size; - - return read_uncompressed(bin_content, bin_size); + return read(bin_content, bin_size); } - Tar read_binary_tar_gz() { - const uint8_t* bin_content = &_binary_input_bin_start; - const int bin_size = (intptr_t) &_binary_input_bin_size; - + static Tar_data decompress_tar() + { + const uint8_t* bin_content = &_binary_input_bin_start; + const int bin_size = (intptr_t) &_binary_input_bin_size; return decompress(bin_content, bin_size); } + static Tar read(const uint8_t* data, const size_t size); + /** dlen is only given if the data is from an Element - see decompress(Element&) */ - Tar decompress(const uint8_t* data, size_t size, unsigned int dlen = 0) { - if (!has_uzlib_init) { - uzlib_init(); - has_uzlib_init = true; - } - - // If not decompressing an Element: - if (dlen == 0) - dlen = decompressed_length(data, size); - - TINF_DATA d; - d.source = data; - - int res = uzlib_gzip_parse_header(&d); - - if (res != TINF_OK) - throw Tar_exception(std::string{"Error parsing header: " + std::to_string(res)}); - - uzlib_uncompress_init(&d, NULL, 0); - - auto dest = std::make_unique(dlen); - - d.dest = dest.get(); - - // decompress byte by byte or any other length - d.destSize = DECOMPRESSION_SIZE; - - // INFO("tar::Reader", "Decompression started - waiting..."); - - do { - res = uzlib_uncompress_chksum(&d); - } while (res == TINF_OK); - - if (res not_eq TINF_DONE) - throw Tar_exception(std::string{"Error during decompression. Res: " + std::to_string(res)}); - - // INFO("tar::Reader", "Decompressed %d bytes", d.dest - dest.get()); - - return read_uncompressed(dest.get(), dlen); - } + static Tar_data decompress(const uint8_t* data, + const size_t size, + unsigned int dlen = 0); /* When have a tar.gz file inside a tar file f.ex. */ - Tar decompress(const Element& element) { - return decompress(element.content(), element.size(), decompressed_length(element.content(), element.size())); + static Tar_data decompress(const Element& element) + { + return decompress(element.content(), element.size(), + decompressed_length(element.content(), element.size())); } - Tar read_uncompressed(const uint8_t* data, size_t size); + static uint32_t checksum(const Element& element) + { return crc32(element.content(), element.size()); } + + static unsigned int decompressed_length(const uint8_t* data, const size_t size); }; // < class Reader diff --git a/lib/mender/include/mender/artifact.hpp b/lib/mender/include/mender/artifact.hpp index 9fa31ffe2b..55b178055b 100644 --- a/lib/mender/include/mender/artifact.hpp +++ b/lib/mender/include/mender/artifact.hpp @@ -44,7 +44,8 @@ namespace mender { int version() const { return version_; } const std::string& format() const { return format_; } const std::string& name() const { return name_; } - const tar::Element& get_update(int index = 0) const { return updates_.at(index).elements().at(0); } + const tar::Tar_data& get_update_blob(int index = 0) const + { return updates_.at(index); } void parse_version(const uint8_t* version); @@ -52,10 +53,9 @@ namespace mender { private: byte_seq data_; - tar::Reader reader_; tar::Tar artifact_; // version, header.tar.gz, data/0000.tar.gz, data/0001.tar.gz, etc. - tar::Tar header_; // unpacked header.tar.gz - std::vector updates_; // unpacked data/0000.tar.gz, data/0001.tar.gz, etc. + tar::Tar_data header_; // unpacked header.tar.gz + std::vector updates_; // unpacked data/0000.tar.gz, data/0001.tar.gz, etc. int version_; // version specified in version file std::string format_; // format specified in version file diff --git a/lib/mender/src/artifact.cpp b/lib/mender/src/artifact.cpp index d8f9235cfe..6d9994b333 100644 --- a/lib/mender/src/artifact.cpp +++ b/lib/mender/src/artifact.cpp @@ -22,7 +22,6 @@ namespace mender { Artifact::Artifact(byte_seq data) : data_(std::move(data)), - reader_{}, artifact_{} { parse(); @@ -35,7 +34,7 @@ namespace mender { // Print and add MENDER_INFO("Artifact", "Parsing data as mender arifact (%u bytes)", (uint32_t)data_.size()); - artifact_ = reader_.read_uncompressed(data_.data(), data_.size()); + artifact_ = Reader::read(data_.data(), data_.size()); auto& elements = artifact_.elements(); MENDER_INFO("Artifact", "Content"); @@ -48,18 +47,15 @@ namespace mender { // Header auto& header_element = elements.at(HEADER); MENDER_INFO2("%s", header_element.name().c_str()); - header_ = reader_.decompress(header_element); + header_ = Reader::decompress(header_element); - auto& header_elements = header_.elements(); + auto header = Reader::read(header_.data(), header_.size()); - for (size_t i = 0; i < header_.elements().size(); i++) { - const std::string header_element_name = header_.elements().at(i).name(); - MENDER_INFO2("\t%s", header_element_name.c_str()); - } + for(auto& h_element : header) + MENDER_INFO2("\t%s", h_element.name().c_str()); // header_info - auto& header_info = header_elements.at(0); - parse_header_info(header_info.content()); + parse_header_info(header.begin()->content()); // Data for (size_t i = DATA_START; i < elements.size(); i++) { @@ -70,15 +66,14 @@ namespace mender { MENDER_INFO2("%s", data_tar_gz.name().c_str()); - auto tar = reader_.decompress(data_tar_gz); + auto tar_data = Reader::decompress(data_tar_gz); - for (auto& data_element : tar.elements()) - { - (void) data_element; - MENDER_INFO2("\t%s", data_element.name().c_str()); - } + // Print content + auto tar = Reader::read(tar_data.data(), tar_data.size()); + for (const auto& d_element : tar) + MENDER_INFO2("\t%s", d_element.name().c_str()); - updates_.push_back(tar); + updates_.push_back(tar_data); } } diff --git a/lib/mender/src/client.cpp b/lib/mender/src/client.cpp index 494ccbdd6f..e71bef0213 100644 --- a/lib/mender/src/client.cpp +++ b/lib/mender/src/client.cpp @@ -258,7 +258,9 @@ namespace mender { // artifact.update() <- this is what liveupdate wants //artifact.verify(); - auto& e = artifact.get_update(0); // returns element with index + const auto& blob = artifact.get_update_blob(0); // returns element with index + auto tar = tar::Reader::read(blob.data(), blob.size()); + const auto& e = *tar.begin(); device_.inventory("artifact_name") = artifact.name(); diff --git a/src/util/tar.cpp b/src/util/tar.cpp index 3d97d68a38..63a9e99371 100644 --- a/src/util/tar.cpp +++ b/src/util/tar.cpp @@ -15,7 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include +#include #include // strtol @@ -59,7 +60,8 @@ std::vector Tar::element_names() const { // -------------------- Reader -------------------- -unsigned int Reader::decompressed_length(const uint8_t* data, size_t size) const { +unsigned int Reader::decompressed_length(const uint8_t* data, const size_t size) +{ unsigned int dlen = data[size - 1]; for (int i = 2; i <= 4; i++) @@ -68,7 +70,7 @@ unsigned int Reader::decompressed_length(const uint8_t* data, size_t size) const return dlen; } -Tar Reader::read_uncompressed(const uint8_t* data, size_t size) { +Tar Reader::read(const uint8_t* data, const size_t size) { if (size % SECTOR_SIZE not_eq 0) throw Tar_exception("Invalid size of tar file"); @@ -97,3 +99,45 @@ Tar Reader::read_uncompressed(const uint8_t* data, size_t size) { return tar; } + +Tar_data Reader::decompress(const uint8_t* data, const size_t size, unsigned int dlen) +{ + if (!has_uzlib_init) { + uzlib_init(); + has_uzlib_init = true; + } + + // If not decompressing an Element: + if (dlen == 0) + dlen = decompressed_length(data, size); + + TINF_DATA d; + d.source = data; + + int res = uzlib_gzip_parse_header(&d); + + if (res != TINF_OK) + throw Tar_exception(std::string{"Error parsing header: " + std::to_string(res)}); + + uzlib_uncompress_init(&d, NULL, 0); + + std::vector dest(dlen); + + d.dest = dest.data(); + + // decompress byte by byte or any other length + d.destSize = DECOMPRESSION_SIZE; + + // INFO("tar::Reader", "Decompression started - waiting..."); + + do { + res = uzlib_uncompress_chksum(&d); + } while (res == TINF_OK); + + if (res not_eq TINF_DONE) + throw Tar_exception(std::string{"Error during decompression. Res: " + std::to_string(res)}); + + // INFO("tar::Reader", "Decompressed %d bytes", d.dest - dest.get()); + Ensures(dest.size() == dlen); + return dest; +} diff --git a/test/util/integration/tar_gz/service.cpp b/test/util/integration/tar_gz/service.cpp index 2b64d9f71e..b30aa4d50e 100644 --- a/test/util/integration/tar_gz/service.cpp +++ b/test/util/integration/tar_gz/service.cpp @@ -51,8 +51,8 @@ void print_content(const Element& element, const std::string& unique) { void Service::start(const std::string&) { - Reader tr; - Tar read_tarfile = tr.read_binary_tar_gz(); // In CMakeLists.txt: set(TARFILE tar_example.tar.gz) + const auto decompressed = Reader::decompress_tar(); // In CMakeLists.txt: set(TARFILE tar_example.tar.gz) + Tar read_tarfile = Reader::read(decompressed.data(), decompressed.size()); // Get the names of all the elements in the tarball auto found_elements = read_tarfile.element_names(); diff --git a/test/util/integration/tar_gz/test.py b/test/util/integration/tar_gz/test.py index 7f645c3a78..2e72626d05 100755 --- a/test/util/integration/tar_gz/test.py +++ b/test/util/integration/tar_gz/test.py @@ -58,7 +58,7 @@ def check_num_outputs(line): # README.md's header vm.on_output("tar_example/l1_f1/l2/README.md - name of README.md", increment_header1) -vm.on_output("Mode of README.md: 0000", increment_header1) +vm.on_output("Mode of README.md: 000", increment_header1) vm.on_output("Uid of README.md: ", increment_header1) vm.on_output("Gid of README.md: ", increment_header1) vm.on_output("Size of README.md: 293", increment_header1) @@ -83,7 +83,7 @@ def check_num_outputs(line): # Folder l2's header and content (no content) vm.on_output("tar_example/l1_f1/l2/ - name of l2", increment_header2) -vm.on_output("Mode of l2: 0000", increment_header2) +vm.on_output("Mode of l2: 000", increment_header2) vm.on_output("Uid of l2: ", increment_header2) vm.on_output("Gid of l2: ", increment_header2) vm.on_output("Size of l2: 0", increment_header2) From ae78bc3f5c2b42978f961c3a56d861bd14955578 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 10 Apr 2018 15:50:13 +0200 Subject: [PATCH 151/723] musl: More work on socketcalls and socket api --- api/posix/fd.hpp | 2 +- api/posix/tcp_fd.hpp | 3 ++- api/posix/udp_fd.hpp | 2 +- src/musl/socketcall.cpp | 35 ++++++++++++++++++++++++++++------- src/posix/tcp_fd.cpp | 24 ++++++++++++------------ src/posix/udp_fd.cpp | 11 ++++------- 6 files changed, 48 insertions(+), 29 deletions(-) diff --git a/api/posix/fd.hpp b/api/posix/fd.hpp index c07661324b..44cc652787 100644 --- a/api/posix/fd.hpp +++ b/api/posix/fd.hpp @@ -49,7 +49,7 @@ class FD { /** SOCKET **/ virtual int accept(struct sockaddr *__restrict__, socklen_t *__restrict__) { return -1; } - virtual int bind(const struct sockaddr *, socklen_t) { return -1; } + virtual long bind(const struct sockaddr *, socklen_t) { return -ENOTSOCK; } virtual int connect(const struct sockaddr *, socklen_t) { return -1; } virtual int getsockopt(int, int, void *__restrict__, socklen_t *__restrict__); virtual int listen(int) { return -1; } diff --git a/api/posix/tcp_fd.hpp b/api/posix/tcp_fd.hpp index 5b7bb4cdc4..4de6aad00f 100644 --- a/api/posix/tcp_fd.hpp +++ b/api/posix/tcp_fd.hpp @@ -38,13 +38,14 @@ class TCP_FD : public SockFD { int close() override; /** SOCKET */ - int bind(const struct sockaddr *, socklen_t) override; + long bind(const struct sockaddr *, socklen_t) override; int listen(int) override; int accept(struct sockaddr *__restrict__, socklen_t *__restrict__) override; int connect(const struct sockaddr *, socklen_t) override; ssize_t send(const void *, size_t, int fl) override; ssize_t recv(void*, size_t, int fl) override; + ssize_t recvfrom(void*, size_t, int fl, struct sockaddr*, socklen_t *) override; int shutdown(int) override; diff --git a/api/posix/udp_fd.hpp b/api/posix/udp_fd.hpp index 5923ea8d6e..42777f5b67 100644 --- a/api/posix/udp_fd.hpp +++ b/api/posix/udp_fd.hpp @@ -45,7 +45,7 @@ class UDP_FD : public SockFD { int close() override; /** SOCKET */ - int bind(const struct sockaddr *, socklen_t) override; + long bind(const struct sockaddr *, socklen_t) override; int connect(const struct sockaddr *, socklen_t) override; ssize_t sendto(const void *, size_t, int, const struct sockaddr *, socklen_t) override; diff --git a/src/musl/socketcall.cpp b/src/musl/socketcall.cpp index 024e54de9d..9d3c17a826 100644 --- a/src/musl/socketcall.cpp +++ b/src/musl/socketcall.cpp @@ -75,48 +75,69 @@ static ssize_t sock_recvfrom(int sockfd, void *buf, size_t len, int flags, return -EBADF; } -extern "C" +static long sock_listen(int sockfd, int backlog) +{ + if(auto* fildes = FD_map::_get(sockfd); fildes) + return fildes->listen(backlog); + + return -EBADF; +} + +static long sock_accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen) +{ + if(auto* fildes = FD_map::_get(sockfd); fildes) + return fildes->accept(addr, addrlen); + + return -EBADF; +} + +extern "C" { long socketcall_socket(int domain, int type, int protocol) { return strace(sock_socket, "socket", domain, type, protocol); } -extern "C" long socketcall_connect(int sockfd, const struct sockaddr *addr, socklen_t addrlen) { return strace(sock_connect, "connect", sockfd, addr, addrlen); } -extern "C" long socketcall_bind(int sockfd, const struct sockaddr *addr, socklen_t addrlen) { return strace(sock_bind, "bind", sockfd, addr, addrlen); } -extern "C" long socketcall_sendmsg(int sockfd, const struct msghdr *msg, int flags) { return strace(sock_sendmsg, "sendmsg", sockfd, msg, flags); } -extern "C" ssize_t socketcall_sendto(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen) { return strace(sock_sendto, "sendto", sockfd, buf, len, flags, dest_addr, addrlen); } -extern "C" ssize_t socketcall_recvfrom(int sockfd, void *buf, size_t len, int flags, struct sockaddr *src_addr, socklen_t *addrlen) { return strace(sock_recvfrom, "recvfrom", sockfd, buf, len, flags, src_addr, addrlen); } -extern "C" +long socketcall_listen(int sockfd, int backlog) +{ + return strace(sock_listen, "listen", sockfd, backlog); +} + +long socketcall_accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen) +{ + return strace(sock_accept, "accept", sockfd, addr, addrlen); +} + long socketcall_shutdown() { return -ENOSYS; } +} // < extern "C" diff --git a/src/posix/tcp_fd.cpp b/src/posix/tcp_fd.cpp index 77b3c681cd..45cfa68e04 100644 --- a/src/posix/tcp_fd.cpp +++ b/src/posix/tcp_fd.cpp @@ -141,6 +141,11 @@ ssize_t TCP_FD::recv(void* dest, size_t len, int flags) return cd->recv(dest, len, flags); } +ssize_t TCP_FD::recvfrom(void* dest, size_t len, int flags, struct sockaddr*, socklen_t*) +{ + return recv(dest, len, flags); +} + int TCP_FD::accept(struct sockaddr *__restrict__ addr, socklen_t *__restrict__ addr_len) { if (!ld) { @@ -157,23 +162,20 @@ int TCP_FD::listen(int backlog) } return ld->listen(backlog); } -int TCP_FD::bind(const struct sockaddr *addr, socklen_t addrlen) +long TCP_FD::bind(const struct sockaddr *addr, socklen_t addrlen) { // - if (cd) { - errno = EINVAL; - return -1; + if (cd or ld) { + return -EINVAL; } // verify socket address if (addrlen != sizeof(sockaddr_in)) { - errno = EINVAL; - return -1; + return -EINVAL; } auto* sin = (sockaddr_in*) addr; // verify its AF_INET if (sin->sin_family != AF_INET) { - errno = EAFNOSUPPORT; - return -1; + return -EAFNOSUPPORT; } // use sin_port for bind // its network order ... so swap that shit: @@ -193,15 +195,13 @@ int TCP_FD::bind(const struct sockaddr *addr, socklen_t addrlen) return 0; } catch (...) { - errno = EADDRINUSE; - return -1; + return -EADDRINUSE; } } int TCP_FD::shutdown(int mode) { if (!cd) { - errno = EINVAL; - return -1; + return -EINVAL; } return cd->shutdown(mode); } diff --git a/src/posix/udp_fd.cpp b/src/posix/udp_fd.cpp index 086d6275b3..95c7a9bab8 100644 --- a/src/posix/udp_fd.cpp +++ b/src/posix/udp_fd.cpp @@ -91,17 +91,15 @@ int UDP_FD::close() { return 0; } -int UDP_FD::bind(const struct sockaddr* address, socklen_t len) +long UDP_FD::bind(const struct sockaddr* address, socklen_t len) { // we can assume this has already been bound since there is a pointer if(UNLIKELY(this->sock != nullptr)) { - errno = EINVAL; - return -1; + return -EINVAL; } // invalid address if(UNLIKELY(len != sizeof(struct sockaddr_in))) { - errno = EINVAL; - return -1; + return -EINVAL; } // Bind const auto port = ((sockaddr_in*)address)->sin_port; @@ -115,8 +113,7 @@ int UDP_FD::bind(const struct sockaddr* address, socklen_t len) } catch(const net::UDP::Port_in_use_exception&) { - errno = EADDRINUSE; - return -1; + return -EADDRINUSE; } } int UDP_FD::connect(const struct sockaddr* address, socklen_t address_len) From 2bb004cab0fcf567aa80f74619c59754fd63a726 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 10 Apr 2018 16:00:51 +0200 Subject: [PATCH 152/723] solo5: just enough WIP to build/boot with musl. --- cmake/post.service.cmake | 2 +- src/kernel/os.cpp | 2 +- src/platform/x86_solo5/kernel_start.cpp | 5 ++--- src/platform/x86_solo5/os.cpp | 9 ++++---- src/platform/x86_solo5/platform.cpp | 3 +++ src/platform/x86_solo5/serial1.cpp | 28 +++++++++++++++++++++---- 6 files changed, 36 insertions(+), 13 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index b382468fe8..ccc1dc5ae5 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -429,7 +429,7 @@ endif(TARFILE) if ("${PLATFORM}" STREQUAL "x86_solo5") - target_link_libraries(service solo5 --whole-archive crtn --no-whole-archive) + target_link_libraries(service solo5) endif() # all the OS and C/C++ libraries + crt end diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index ce24d1101e..5d5a952ad0 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -94,7 +94,7 @@ extern char __service_ctors_end; extern char __plugin_ctors_start; extern char __plugin_ctors_end; -static int __run_ctors(uintptr_t begin, uintptr_t end) +int __run_ctors(uintptr_t begin, uintptr_t end) { int i = 0; for (; begin < end; begin += sizeof(void(*)()), i++) { diff --git a/src/platform/x86_solo5/kernel_start.cpp b/src/platform/x86_solo5/kernel_start.cpp index f665507bb6..09d6d5b224 100644 --- a/src/platform/x86_solo5/kernel_start.cpp +++ b/src/platform/x86_solo5/kernel_start.cpp @@ -21,8 +21,8 @@ extern "C" { uintptr_t _move_symbols(uintptr_t loc); void _init_bss(); void _init_heap(uintptr_t); - void _init_c_runtime(); void _init_syscalls(); + void _init_elf_parser(); uintptr_t _end; void set_stack(); void* get_cpu_ebp(); @@ -48,8 +48,7 @@ extern "C" { // so might be OK (for now). _init_heap(free_mem_begin); - //Initialize stack-unwinder, call global constructors etc. - _init_c_runtime(); + _init_elf_parser(); // Initialize system calls _init_syscalls(); diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index 42647393e3..8ad31cd4bd 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -14,7 +14,9 @@ extern "C" { static uint64_t os_cycles_hlt = 0; extern "C" void* get_cpu_esp(); -extern "C" void __libc_init_array(); +extern char __init_array_start; +extern char __init_array_end; +int __run_ctors(uintptr_t begin, uintptr_t end); extern uintptr_t heap_begin; extern uintptr_t heap_end; extern uintptr_t _start; @@ -94,9 +96,8 @@ void OS::start(char* _cmdline, uintptr_t mem_size) OS::heap_max_ = OS::memory_end_; // Call global ctors - PROFILE("Global constructors"); - __libc_init_array(); - + PROFILE("Global kernel constructors"); + __run_ctors((uintptr_t)&__init_array_start, (uintptr_t)&__init_array_end); PROFILE("Memory map"); // Assign memory ranges used by the kernel diff --git a/src/platform/x86_solo5/platform.cpp b/src/platform/x86_solo5/platform.cpp index 4d80081cc8..e93188d2d0 100644 --- a/src/platform/x86_solo5/platform.cpp +++ b/src/platform/x86_solo5/platform.cpp @@ -1,6 +1,9 @@ #include #include +bool __libc_initialized = false; + + void __arch_poweroff() { while (true) asm ("cli; hlt"); diff --git a/src/platform/x86_solo5/serial1.cpp b/src/platform/x86_solo5/serial1.cpp index a519ad4cbd..d90671f503 100644 --- a/src/platform/x86_solo5/serial1.cpp +++ b/src/platform/x86_solo5/serial1.cpp @@ -1,14 +1,34 @@ -#include +#include +#include +#include extern "C" { #include -} -extern "C" void __init_serial1() {} -extern "C" void __serial_print1(const char* cstr) { solo5_console_write(cstr, strlen(cstr)); } +void __serial_print(const char* str, size_t len) +{ + solo5_console_write(str, len); +} + +void kprint(const char* c){ + __serial_print1(c); +} + +void kprintf(const char* format, ...) +{ + char buf[8192]; + va_list aptr; + va_start(aptr, format); + vsnprintf(buf, sizeof(buf), format, aptr); + __serial_print1(buf); + va_end(aptr); +} + + +} From 53a1b9693e2c017da5a68a377a39acba31b2214b Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 10 Apr 2018 16:22:40 +0200 Subject: [PATCH 153/723] test: Fix side-effect in RNG close assert --- test/kernel/integration/rng/CMakeLists.txt | 6 ++---- test/kernel/integration/rng/service.cpp | 11 +++++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/test/kernel/integration/rng/CMakeLists.txt b/test/kernel/integration/rng/CMakeLists.txt index 571962fa47..0082ec77db 100644 --- a/test/kernel/integration/rng/CMakeLists.txt +++ b/test/kernel/integration/rng/CMakeLists.txt @@ -15,15 +15,13 @@ set(SERVICE_NAME "RNG Test Service") # Name of your service binary set(BINARY "rng_jesus") -# Maximum memory can be hard-coded into the binary -set(MAX_MEM 128) - - # Source files to be linked with OS library parts to form bootable image set(SOURCES service.cpp # ...add more here ) +#set(DRIVERS boot_logger) + set(PLUGINS vfs) # include service build script diff --git a/test/kernel/integration/rng/service.cpp b/test/kernel/integration/rng/service.cpp index d2979a1689..4791cabfbd 100644 --- a/test/kernel/integration/rng/service.cpp +++ b/test/kernel/integration/rng/service.cpp @@ -23,19 +23,22 @@ #include #include // hist -const size_t BUFLEN = 4096; -void Service::start(const std::string&) +static const size_t BUFLEN = 4096; +void Service::start() { INFO("POSIX", "Operating on fd \"dev/(u)random\""); int fd = open("/dev/urandom", O_RDONLY); CHECKSERT(fd > 0, "open /dev/urandom returns fd"); + printf("fd: %d\n", fd); + + // close is *not* ignored! + int res = close(fd); + CHECKSERT(res == 0, "close does not return error"); int rand_fd = open("/dev/random", O_RDONLY); CHECKSERT(rand_fd > 0, "open /dev/random returns fd"); - CHECKSERT(0 == close(rand_fd), "close does not return error"); // ignored - uint8_t a[BUFLEN]; uint8_t b[BUFLEN]; From 1b971e8b5dee29942c7694d6ef00eb179d44f856 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 10 Apr 2018 16:23:22 +0200 Subject: [PATCH 154/723] musl: Always erase open fd from FD_map --- src/musl/close.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/musl/close.cpp b/src/musl/close.cpp index b1a2637736..bc3b782db7 100644 --- a/src/musl/close.cpp +++ b/src/musl/close.cpp @@ -6,10 +6,8 @@ static long sys_close(int fd) if(auto* fildes = FD_map::_get(fd); fildes) { long res = fildes->close(); - if(res != 0) - return res; FD_map::close(fd); - return 0; + return res; } return -EBADF; } From 5b35b73005314d05cbadeb6e1e1d2c415271288c Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 10 Apr 2018 16:27:06 +0200 Subject: [PATCH 155/723] vmrunner: Don't use user networking as default --- vmrunner/vm.schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmrunner/vm.schema.json b/vmrunner/vm.schema.json index 9b5fd91f01..705317567d 100644 --- a/vmrunner/vm.schema.json +++ b/vmrunner/vm.schema.json @@ -75,7 +75,7 @@ "properties" : { "device" : { "type" : "string" }, "name" : { "type" : "string" }, - "backend" : { "enum" : ["tap", "user"], "default" : "user" } + "backend" : { "enum" : ["tap", "user"], "default" : "tap" } }, "required" : ["device"] From bb5ad0ec1d2780a945894abc5552bd165b94baf5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 10 Apr 2018 17:47:30 +0200 Subject: [PATCH 156/723] diskbuilder: Allow empty files --- diskimagebuild/filetree.cpp | 9 ++++++--- diskimagebuild/writer.cpp | 7 +++++-- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/diskimagebuild/filetree.cpp b/diskimagebuild/filetree.cpp index 202e8e94bf..3e6f365494 100644 --- a/diskimagebuild/filetree.cpp +++ b/diskimagebuild/filetree.cpp @@ -21,9 +21,12 @@ File::File(const char* path) rewind(f); this->data = std::unique_ptr (new char[size], std::default_delete ()); - size_t actual = fread(this->data.get(), this->size, 1, f); - if (actual != 1) { - throw std::runtime_error("diskbuilder: Could not read from file " + std::string(path)); + if (this->size > 0) + { + size_t actual = fread(this->data.get(), this->size, 1, f); + if (actual != 1) { + throw std::runtime_error("diskbuilder: Could not read from file " + std::string(path)); + } } fclose(f); } diff --git a/diskimagebuild/writer.cpp b/diskimagebuild/writer.cpp index 7cc84abdce..cf4db228c4 100644 --- a/diskimagebuild/writer.cpp +++ b/diskimagebuild/writer.cpp @@ -280,8 +280,11 @@ long File::write(FileSys&, FILE* file, long pos) const { //printf("writing file to %ld with size %u\n", pos, this->size); fseek(file, pos, SEEK_SET); - int count = fwrite(data.get(), this->size, 1, file); - assert(count == 1); + if (this->size > 0) + { + int count = fwrite(data.get(), this->size, 1, file); + assert(count == 1); + } // write zeroes to remainder int rem = SECT_SIZE - (this->size & (SECT_SIZE-1)); fwrite("\0", 1, rem, file); From 5e95ada54ae275fea9e5005e49f48447ceb40748 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 10 Apr 2018 17:47:56 +0200 Subject: [PATCH 157/723] cmake: Install OpenSSL includes correctly --- cmake/openssl.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/openssl.cmake b/cmake/openssl.cmake index 7556812f7e..925a81da84 100644 --- a/cmake/openssl.cmake +++ b/cmake/openssl.cmake @@ -26,5 +26,5 @@ if(${ARCH} STREQUAL "x86_64") install(FILES ${OPENSSL_LIB_CRYPTO} DESTINATION includeos/${ARCH}/lib) install(FILES ${OPENSSL_LIB_SSL} DESTINATION includeos/${ARCH}/lib) - install(DIRECTORY ${OPENSSL_INCLUDE} DESTINATION includeos/${ARCH}/include) + install(DIRECTORY ${OPENSSL_INCLUDE} DESTINATION includeos/${ARCH}) endif() From be46daf190ffdaf0214588dc207837f2499cba39 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 10 Apr 2018 17:51:23 +0200 Subject: [PATCH 158/723] test: Update util Tar test --- test/util/integration/tar/service.cpp | 3 +-- test/util/integration/tar/test.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/util/integration/tar/service.cpp b/test/util/integration/tar/service.cpp index ce8c50f2fb..3c4f546154 100644 --- a/test/util/integration/tar/service.cpp +++ b/test/util/integration/tar/service.cpp @@ -51,8 +51,7 @@ void print_content(const Element& element, const std::string& unique) { void Service::start(const std::string&) { - Reader tr; - Tar read_tarfile = tr.read_binary_tar(); // In CMakeLists.txt: set(TARFILE tar_example.tar) + Tar read_tarfile = Reader::read_tar(); // In CMakeLists.txt: set(TARFILE tar_example.tar) // Get the names of all the elements in the tarball auto found_elements = read_tarfile.element_names(); diff --git a/test/util/integration/tar/test.py b/test/util/integration/tar/test.py index a7eafcc398..332f0fd668 100755 --- a/test/util/integration/tar/test.py +++ b/test/util/integration/tar/test.py @@ -86,7 +86,7 @@ def check_num_outputs(line): # Folder l2's header and content (no content) vm.on_output("tar_example/l1_f1/l2/ - name of l2", increment_header2) -vm.on_output("Mode of l2: 0000775", increment_header2) +vm.on_output("Mode of l2: 000", increment_header2) # TODO: check for not null vm.on_output("Uid of l2: ", increment_header2) From dacfac23c7235c8529f95c0b4b1b9a8e2ef5706b Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 10 Apr 2018 18:46:04 +0200 Subject: [PATCH 159/723] test: change plugin test to run directly in ctors --- .../integration/plugin_init/plugin1.cpp | 20 ++++++++++++------- .../integration/plugin_init/plugin2.cpp | 8 ++------ .../integration/plugin_init/plugin3.cpp | 8 ++------ 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/test/kernel/integration/plugin_init/plugin1.cpp b/test/kernel/integration/plugin_init/plugin1.cpp index 9f29b0c096..88957c2430 100644 --- a/test/kernel/integration/plugin_init/plugin1.cpp +++ b/test/kernel/integration/plugin_init/plugin1.cpp @@ -1,16 +1,22 @@ #include +extern bool example_plugin_registered; +extern bool example_plugin_run; int my_plugin_functions = 0; int f1_data = 0; -void func1(){ +__attribute__((constructor)) +static void func1(){ + + + // NOTE: since this plugin is a part of the service, its ctors + // will be called after the installed plugin ctors. Therefore we can + // depend on installed plugins being run / registered. + + CHECKSERT(example_plugin_registered, "Example plugin registered"); + CHECKSERT(example_plugin_run, "Example plugin ran"); + INFO("Plugin 1","initialization function 1"); f1_data = 0xf1; my_plugin_functions++; } - -struct Autoreg { - Autoreg() { - OS::register_plugin(func1, "Plugin 1"); - } -} autoregister; diff --git a/test/kernel/integration/plugin_init/plugin2.cpp b/test/kernel/integration/plugin_init/plugin2.cpp index 01ecd6a3ca..3864f12d56 100644 --- a/test/kernel/integration/plugin_init/plugin2.cpp +++ b/test/kernel/integration/plugin_init/plugin2.cpp @@ -3,13 +3,9 @@ extern int my_plugin_functions; int f2_data = 0; -void func2(){ +__attribute__((constructor)) +static void func2(){ INFO("Plugin 2","initialization function 2"); f2_data = 0xf2; my_plugin_functions++; } - -__attribute__((constructor)) -void autoregister(){ - OS::register_plugin(func2, "Plugin 2"); -} diff --git a/test/kernel/integration/plugin_init/plugin3.cpp b/test/kernel/integration/plugin_init/plugin3.cpp index f1ceb4f10a..0b27c080b6 100644 --- a/test/kernel/integration/plugin_init/plugin3.cpp +++ b/test/kernel/integration/plugin_init/plugin3.cpp @@ -13,13 +13,9 @@ class myexcept : public std::exception { } }; -void func3(){ +__attribute__((constructor)) +static void func3(){ INFO("Plugin 3","initialization function 3"); my_plugin_functions++; f3_data = 0xf3; } - -__attribute__((constructor)) -void autoregister3(){ - OS::register_plugin(func3, "Plugin 3"); -} From 4f81569cbff4d76f3a3d13d30e4bb81d2f97e746 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 11 Apr 2018 09:32:12 +0200 Subject: [PATCH 160/723] test: Fix websocket test, use SMP::Array --- test/net/integration/websocket/service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net/integration/websocket/service.cpp b/test/net/integration/websocket/service.cpp index 4a462c81ec..3ce4d7d1d8 100644 --- a/test/net/integration/websocket/service.cpp +++ b/test/net/integration/websocket/service.cpp @@ -13,7 +13,7 @@ struct alignas(SMP_ALIGN) HTTP_server // websocket clients std::deque clients; }; -static SMP_ARRAY httpd; +static SMP::Array httpd; static net::WebSocket_ptr& new_client(net::WebSocket_ptr socket) { From 752984f360e6076832f8f52de7d6a3956a285bd8 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 11 Apr 2018 09:59:50 +0200 Subject: [PATCH 161/723] test: make microLB always kill node.js and start after init --- test/net/integration/microLB/service.cpp | 1 + test/net/integration/microLB/test.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/test/net/integration/microLB/service.cpp b/test/net/integration/microLB/service.cpp index 606eb87c26..d76439a1e6 100644 --- a/test/net/integration/microLB/service.cpp +++ b/test/net/integration/microLB/service.cpp @@ -21,4 +21,5 @@ void Service::start() { static auto* balancer = microLB::Balancer::from_config(); + printf("MicroLB ready for test\n"); } diff --git a/test/net/integration/microLB/test.py b/test/net/integration/microLB/test.py index 0c1cbdd41c..c410df625c 100755 --- a/test/net/integration/microLB/test.py +++ b/test/net/integration/microLB/test.py @@ -4,6 +4,7 @@ import sys import subprocess import thread +import atexit includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) @@ -21,6 +22,7 @@ def validateRequest(expected = ""): pro = subprocess.Popen(["node", "server.js"], stdout=subprocess.PIPE) def startBenchmark(line): + print " starting test " assert validateRequest("6001") assert validateRequest("6002") assert validateRequest("6003") @@ -30,16 +32,21 @@ def startBenchmark(line): assert validateRequest("6002") assert validateRequest("6003") assert validateRequest("6004") - # stop nodeJS - pro.kill() vm._on_success("SUCCESS") return True +@atexit.register +def cleanup(): + print " Stopping node server" + # stop nodeJS + pro.kill() + + # Get an auto-created VM from the vmrunner vm = vmrunner.vms[0] # Add custom event for testing server -vm.on_output("Running [ Micro Load Balancer ]", startBenchmark) +vm.on_output("MicroLB ready for test", startBenchmark) # Boot the VM, taking a timeout as parameter vm.cmake().boot(20).clean() From 8531ecae6cc0bc587698cb862350b75841174562 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 11 Apr 2018 11:12:48 +0200 Subject: [PATCH 162/723] net: Fix size print on Listener::to_string --- src/net/tcp/listener.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/tcp/listener.cpp b/src/net/tcp/listener.cpp index 253f061073..bb34da7c95 100644 --- a/src/net/tcp/listener.cpp +++ b/src/net/tcp/listener.cpp @@ -147,7 +147,7 @@ void Listener::close() { std::string Listener::to_string() const { char buffer[256]; int len = snprintf(buffer, sizeof(buffer), - "[%s] SynQueue (%ld) ", local_.to_string().c_str(), syn_queue_.size()); + "[%s] SynQueue (%zu) ", local_.to_string().c_str(), syn_queue_.size()); std::string str(buffer, len); // add syn queue for(auto& conn : syn_queue_) From 290858a0c2fdb63611cd3ad5ea9a0c5dfdc14455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 11 Apr 2018 11:59:03 +0200 Subject: [PATCH 163/723] musl: improved TCP socket, works in happy day scenarios --- api/posix/fd.hpp | 6 +- api/posix/tcp_fd.hpp | 20 +++--- api/posix/udp_fd.hpp | 2 +- src/musl/socketcall.cpp | 12 +++- src/posix/tcp_fd.cpp | 145 ++++++++++++++++++++++------------------ src/posix/udp_fd.cpp | 2 +- 6 files changed, 107 insertions(+), 80 deletions(-) diff --git a/api/posix/fd.hpp b/api/posix/fd.hpp index 44cc652787..454151c797 100644 --- a/api/posix/fd.hpp +++ b/api/posix/fd.hpp @@ -48,11 +48,11 @@ class FD { virtual int ioctl(int, void*); /** SOCKET **/ - virtual int accept(struct sockaddr *__restrict__, socklen_t *__restrict__) { return -1; } + virtual long accept(struct sockaddr *__restrict__, socklen_t *__restrict__) { return -ENOTSOCK; } virtual long bind(const struct sockaddr *, socklen_t) { return -ENOTSOCK; } - virtual int connect(const struct sockaddr *, socklen_t) { return -1; } + virtual long connect(const struct sockaddr *, socklen_t) { return-ENOTSOCK; } virtual int getsockopt(int, int, void *__restrict__, socklen_t *__restrict__); - virtual int listen(int) { return -1; } + virtual long listen(int) { return -ENOTSOCK; } virtual ssize_t recv(void *, size_t, int) { return 0; } virtual ssize_t recvfrom(void *__restrict__, size_t, int, struct sockaddr *__restrict__, socklen_t *__restrict__) { return 0; } virtual ssize_t recvmsg(struct msghdr *, int) { return 0; } diff --git a/api/posix/tcp_fd.hpp b/api/posix/tcp_fd.hpp index 4de6aad00f..c73f7ff1d3 100644 --- a/api/posix/tcp_fd.hpp +++ b/api/posix/tcp_fd.hpp @@ -39,11 +39,13 @@ class TCP_FD : public SockFD { /** SOCKET */ long bind(const struct sockaddr *, socklen_t) override; - int listen(int) override; - int accept(struct sockaddr *__restrict__, socklen_t *__restrict__) override; - int connect(const struct sockaddr *, socklen_t) override; + long listen(int) override; + long accept(struct sockaddr *__restrict__, socklen_t *__restrict__) override; + long connect(const struct sockaddr *, socklen_t) override; ssize_t send(const void *, size_t, int fl) override; + ssize_t sendto(const void *, size_t, int fl, + const struct sockaddr*, socklen_t) override; ssize_t recv(void*, size_t, int fl) override; ssize_t recvfrom(void*, size_t, int fl, struct sockaddr*, socklen_t *) override; @@ -65,7 +67,7 @@ class TCP_FD : public SockFD { ~TCP_FD() {} private: - TCP_FD_Conn* cd = nullptr; + std::unique_ptr cd = nullptr; TCP_FD_Listen* ld = nullptr; friend struct TCP_FD_Listen; @@ -73,8 +75,7 @@ class TCP_FD : public SockFD { struct TCP_FD_Conn { - TCP_FD_Conn(net::tcp::Connection_ptr c) - : conn(c) {} + TCP_FD_Conn(net::tcp::Connection_ptr c); void recv_to_ringbuffer(net::tcp::buffer_t); void set_default_read(); @@ -88,6 +89,7 @@ struct TCP_FD_Conn net::tcp::Connection_ptr conn; FixedRingBuffer<16384> readq; + bool recv_disc = false; }; struct TCP_FD_Listen @@ -97,14 +99,14 @@ struct TCP_FD_Listen {} int close(); - int listen(int); - int accept(struct sockaddr *__restrict__, socklen_t *__restrict__); + long listen(int); + long accept(struct sockaddr *__restrict__, socklen_t *__restrict__); int shutdown(int); std::string to_string() const { return listener.to_string(); } net::tcp::Listener& listener; - std::deque connq; + std::deque> connq; }; inline net::tcp::Connection_ptr TCP_FD::get_connection() noexcept { diff --git a/api/posix/udp_fd.hpp b/api/posix/udp_fd.hpp index 42777f5b67..a0c1c3e98e 100644 --- a/api/posix/udp_fd.hpp +++ b/api/posix/udp_fd.hpp @@ -46,7 +46,7 @@ class UDP_FD : public SockFD { /** SOCKET */ long bind(const struct sockaddr *, socklen_t) override; - int connect(const struct sockaddr *, socklen_t) override; + long connect(const struct sockaddr *, socklen_t) override; ssize_t sendto(const void *, size_t, int, const struct sockaddr *, socklen_t) override; diff --git a/src/musl/socketcall.cpp b/src/musl/socketcall.cpp index 9d3c17a826..9015f66031 100644 --- a/src/musl/socketcall.cpp +++ b/src/musl/socketcall.cpp @@ -91,6 +91,14 @@ static long sock_accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen) return -EBADF; } +static long sock_shutdown(int sockfd, int how) +{ + if(auto* fildes = FD_map::_get(sockfd); fildes) + return fildes->shutdown(how); + + return -EBADF; +} + extern "C" { long socketcall_socket(int domain, int type, int protocol) { @@ -136,8 +144,8 @@ long socketcall_accept(int sockfd, struct sockaddr *addr, socklen_t *addrlen) return strace(sock_accept, "accept", sockfd, addr, addrlen); } -long socketcall_shutdown() +long socketcall_shutdown(int sockfd, int how) { - return -ENOSYS; + return strace(sock_shutdown, "shutdown", sockfd, how); } } // < extern "C" diff --git a/src/posix/tcp_fd.cpp b/src/posix/tcp_fd.cpp index 45cfa68e04..3dcb0295c0 100644 --- a/src/posix/tcp_fd.cpp +++ b/src/posix/tcp_fd.cpp @@ -20,7 +20,7 @@ #include #include -#define POSIX_STRACE +//#define POSIX_STRACE #ifdef POSIX_STRACE #define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else @@ -49,7 +49,7 @@ int TCP_FD::close() if (this->cd) { PRINT("TCP: close(%s)\n", cd->to_string().c_str()); int ret = cd->close(); - delete cd; cd = nullptr; + cd = nullptr; return ret; } // listener @@ -59,84 +59,89 @@ int TCP_FD::close() delete ld; ld = nullptr; return ret; } - errno = EBADF; - return -1; + return -EBADF; } -int TCP_FD::connect(const struct sockaddr* address, socklen_t address_len) +long TCP_FD::connect(const struct sockaddr* address, socklen_t address_len) { if (is_listener()) { PRINT("TCP: connect(%s) failed\n", ld->to_string().c_str()); // already listening on port - errno = EINVAL; - return -1; + return -EINVAL; } if (this->cd) { PRINT("TCP: connect(%s) failed\n", cd->to_string().c_str()); // if its straight-up connected, return that if (cd->conn->is_connected()) { - errno = EISCONN; - return -1; + return -EISCONN; } // if the connection isn't closed, we can just assume its being used already if (!cd->conn->is_closed()) { - errno = EALREADY; - return -1; + return -EALREADY; } } if (address_len != sizeof(sockaddr_in)) { - errno = EINVAL; // checkme? - return -1; + return -EINVAL; // checkme? } auto* inaddr = (sockaddr_in*) address; - auto addr = ip4::Addr(inaddr->sin_addr.s_addr); + auto addr = ip4::Addr(::ntohl(inaddr->sin_addr.s_addr)); auto port = ::htons(inaddr->sin_port); PRINT("TCP: connect(%s:%u)\n", addr.to_string().c_str(), port); auto outgoing = net_stack().tcp().connect({addr, port}); + + bool refused = false; + outgoing->on_connect([&refused](auto conn) { + refused = (conn == nullptr); + }); // O_NONBLOCK is set for the file descriptor for the socket and the connection // cannot be immediately established; the connection shall be established asynchronously. if (this->is_blocking() == false) { - errno = EINPROGRESS; - return -1; + return -EINPROGRESS; } // wait for connection state to change - while (not (outgoing->is_connected() || outgoing->is_closing() || outgoing->is_closed())) { + while (not (outgoing->is_connected() or + outgoing->is_closing() or + outgoing->is_closed() or + refused)) + { OS::block(); } // set connection whether good or bad if (outgoing->is_connected()) { - // out with the old - delete this->cd; - // in with the new - this->cd = new TCP_FD_Conn(outgoing); + // out with the old, in with the new + this->cd = std::make_unique(outgoing); cd->set_default_read(); return 0; } // failed to connect // TODO: try to distinguish the reason for connection failure - errno = ECONNREFUSED; - return -1; + return -ECONNREFUSED; } ssize_t TCP_FD::send(const void* data, size_t len, int fmt) { if (!cd) { - errno = EINVAL; - return -1; + return -EINVAL; } return cd->send(data, len, fmt); } +ssize_t TCP_FD::sendto(const void* data, size_t len, int fmt, + const struct sockaddr* dest_addr, socklen_t dest_len) +{ + (void) dest_addr; + (void) dest_len; + return send(data, len, fmt); +} ssize_t TCP_FD::recv(void* dest, size_t len, int flags) { if (!cd) { - errno = EINVAL; - return -1; + return -EINVAL; } return cd->recv(dest, len, flags); } @@ -146,19 +151,17 @@ ssize_t TCP_FD::recvfrom(void* dest, size_t len, int flags, struct sockaddr*, so return recv(dest, len, flags); } -int TCP_FD::accept(struct sockaddr *__restrict__ addr, socklen_t *__restrict__ addr_len) +long TCP_FD::accept(struct sockaddr *__restrict__ addr, socklen_t *__restrict__ addr_len) { if (!ld) { - errno = EINVAL; - return -1; + return -EINVAL; } return ld->accept(addr, addr_len); } -int TCP_FD::listen(int backlog) +long TCP_FD::listen(int backlog) { if (!ld) { - errno = EINVAL; - return -1; + return -EINVAL; } return ld->listen(backlog); } @@ -211,7 +214,7 @@ int TCP_FD::shutdown(int mode) TCP_FD::on_read_func TCP_FD::get_default_read_func() { if (cd) { - return {cd, &TCP_FD_Conn::recv_to_ringbuffer}; + return {cd.get(), &TCP_FD_Conn::recv_to_ringbuffer}; } if (ld) { return @@ -231,6 +234,22 @@ TCP_FD::on_except_func TCP_FD::get_default_except_func() } /// socket as connection +TCP_FD_Conn::TCP_FD_Conn(net::tcp::Connection_ptr c) + : conn{std::move(c)} +{ + assert(conn != nullptr); + set_default_read(); + + conn->on_disconnect([this](auto self, auto reason) { + this->recv_disc = true; + (void) reason; + //printf("dc: %s - %s\n", reason.to_string().c_str(), self->to_string().c_str()); + // do nothing, avoid close + // net::tcp::Connection::Disconnect::CLOSING + if(not self->is_connected()) + self->close(); + }); +} void TCP_FD_Conn::recv_to_ringbuffer(net::tcp::buffer_t buffer) { @@ -246,14 +265,13 @@ void TCP_FD_Conn::recv_to_ringbuffer(net::tcp::buffer_t buffer) } void TCP_FD_Conn::set_default_read() { - // readq buffering (4kb at a time) - conn->on_read(4096, {this, &TCP_FD_Conn::recv_to_ringbuffer}); + // readq buffering (16kb at a time) + conn->on_read(16438, {this, &TCP_FD_Conn::recv_to_ringbuffer}); } ssize_t TCP_FD_Conn::send(const void* data, size_t len, int) { if (not conn->is_connected()) { - errno = ENOTCONN; - return -1; + return -ENOTCONN; } bool written = false; @@ -272,16 +290,13 @@ ssize_t TCP_FD_Conn::send(const void* data, size_t len, int) } ssize_t TCP_FD_Conn::recv(void* dest, size_t len, int) { - // if the connection is closed or closing: read returns 0 - if (conn->is_closed() || conn->is_closing()) return 0; - if (not conn->is_connected()) { - errno = ENOTCONN; - return -1; - } // read some bytes from readq int bytes = readq.read((char*) dest, len); if (bytes) return bytes; + if(this->recv_disc) + return 0; + bool done = false; bytes = 0; @@ -315,9 +330,8 @@ int TCP_FD_Conn::close() } int TCP_FD_Conn::shutdown(int mode) { - if (not conn->is_connected()) { - errno = ENOTCONN; - return -1; + if (conn->is_closed()) { + return -ENOTCONN; } switch (mode) { case SHUT_RDWR: @@ -330,50 +344,53 @@ int TCP_FD_Conn::shutdown(int mode) printf("Ignoring shutdown(SHUT_WR)\n"); return 0; default: - errno = EINVAL; - return -1; + return -EINVAL; } } /// socket as listener -int TCP_FD_Listen::listen(int backlog) +long TCP_FD_Listen::listen(int backlog) { listener.on_connect( [this, backlog] (net::tcp::Connection_ptr conn) { + if(UNLIKELY(not conn)) + return; + // remove oldest if full if ((int) this->connq.size() >= backlog) this->connq.pop_back(); // new connection - this->connq.push_front(conn); + this->connq.push_front(std::make_unique(conn)); /// if someone is blocking they should be leaving right about now }); return 0; } -int TCP_FD_Listen::accept(struct sockaddr *__restrict__ addr, socklen_t *__restrict__ addr_len) +long TCP_FD_Listen::accept(struct sockaddr *__restrict__ addr, socklen_t *__restrict__ addr_len) { // block until connection appears while (connq.empty()) { OS::block(); } // retrieve connection from queue - auto sock = connq.back(); + auto sock = std::move(connq.back()); connq.pop_back(); - // make sure socket is connected, as promised - if (not sock->is_connected()) { - errno = ENOTCONN; - return -1; - } + + assert(sock != nullptr); // create connected TCP socket auto& fd = FD_map::_open(); - fd.cd = new TCP_FD_Conn(sock); + fd.cd = std::move(sock); // set address and length - auto* sin = (sockaddr_in*) addr; - sin->sin_family = AF_INET; - sin->sin_port = sock->remote().port(); - sin->sin_addr.s_addr = sock->remote().address().whole; - *addr_len = sizeof(sockaddr_in); + if(addr != nullptr and addr_len != nullptr) + { + auto& conn = fd.cd->conn; + auto* sin = (sockaddr_in*) addr; + sin->sin_family = AF_INET; + sin->sin_port = conn->remote().port(); + sin->sin_addr.s_addr = conn->remote().address().whole; + *addr_len = sizeof(sockaddr_in); + } // return socket return fd.get_id(); } diff --git a/src/posix/udp_fd.cpp b/src/posix/udp_fd.cpp index 95c7a9bab8..10d17b7a1b 100644 --- a/src/posix/udp_fd.cpp +++ b/src/posix/udp_fd.cpp @@ -116,7 +116,7 @@ long UDP_FD::bind(const struct sockaddr* address, socklen_t len) return -EADDRINUSE; } } -int UDP_FD::connect(const struct sockaddr* address, socklen_t address_len) +long UDP_FD::connect(const struct sockaddr* address, socklen_t address_len) { if (UNLIKELY(address_len < sizeof(struct sockaddr_in))) { return -EINVAL; From ceefd5e18ff6833412699a75a0e909fb8519e526 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 11 Apr 2018 12:02:14 +0200 Subject: [PATCH 164/723] test: Added bare bone TCP posix socket test --- test/posix/integration/tcp/CMakeLists.txt | 30 ++++++ test/posix/integration/tcp/README.md | 3 + test/posix/integration/tcp/service.cpp | 124 ++++++++++++++++++++++ test/posix/integration/tcp/test.py | 70 ++++++++++++ test/posix/integration/tcp/vm.json | 6 ++ 5 files changed, 233 insertions(+) create mode 100644 test/posix/integration/tcp/CMakeLists.txt create mode 100644 test/posix/integration/tcp/README.md create mode 100644 test/posix/integration/tcp/service.cpp create mode 100755 test/posix/integration/tcp/test.py create mode 100644 test/posix/integration/tcp/vm.json diff --git a/test/posix/integration/tcp/CMakeLists.txt b/test/posix/integration/tcp/CMakeLists.txt new file mode 100644 index 0000000000..96946b1713 --- /dev/null +++ b/test/posix/integration/tcp/CMakeLists.txt @@ -0,0 +1,30 @@ +cmake_minimum_required(VERSION 2.8.9) + +# IncludeOS install location +if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) + set(ENV{INCLUDEOS_PREFIX} /usr/local) +endif() + +include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) + +project (test_posix_tcp) + +# Human-readable name of your service +set(SERVICE_NAME "POSIX TCP Test Service") + +# Name of your service binary +set(BINARY "test_posix_tcp") + +# Source files to be linked with OS library parts to form bootable image +set(SOURCES + service.cpp # ...add more here + ) + +# DRIVERS / PLUGINS: + +set(DRIVERS + virtionet # Virtio networking + ) + +# include service build script +include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/posix/integration/tcp/README.md b/test/posix/integration/tcp/README.md new file mode 100644 index 0000000000..c0ec479db3 --- /dev/null +++ b/test/posix/integration/tcp/README.md @@ -0,0 +1,3 @@ +# POSIX TCP Test + +Testing POSIX TCP implementation. VERY bare bones. diff --git a/test/posix/integration/tcp/service.cpp b/test/posix/integration/tcp/service.cpp new file mode 100644 index 0000000000..face523028 --- /dev/null +++ b/test/posix/integration/tcp/service.cpp @@ -0,0 +1,124 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +const uint16_t PORT = 1042; +const uint16_t OUT_PORT = 4242; +const uint16_t BUFSIZE = 2048; + +int main() +{ + auto&& inet = net::Inet4::ifconfig({ 10, 0, 0, 51 }, // IP + { 255, 255, 0, 0 }, // Netmask + { 10, 0, 0, 4 }); // Gateway + + INFO("TCP Socket", "bind(%u)", PORT); + + int fd = socket(AF_INET, SOCK_STREAM, 0); + CHECKSERT(fd > 0, "Socket was created"); + + /* local address */ + struct sockaddr_in myaddr; + + memset((char *)&myaddr, 0, sizeof(myaddr)); + myaddr.sin_family = AF_INET; + myaddr.sin_addr.s_addr = htonl(INADDR_ANY); + myaddr.sin_port = htons(PORT); + + int res = bind(fd, (struct sockaddr *)&myaddr, sizeof(myaddr)); + CHECKSERT(res == 0, "Socket was bound to port %u", PORT); + + res = bind(fd, (struct sockaddr *)&myaddr, sizeof(myaddr)); + printf("err: %i %s res: %i\n", errno, strerror(errno), res); + CHECKSERT(res < 0 && errno == EINVAL, "Fails when already bound (EINVAL)"); + + res = bind(socket(AF_INET, SOCK_STREAM, 0), (struct sockaddr *)&myaddr, 1ul); + CHECKSERT(res < 0 && errno == EINVAL, "Fails when address is invalid (EINVAL)"); + + res = bind(socket(AF_INET, SOCK_STREAM, 0), (struct sockaddr *)&myaddr, sizeof(myaddr)); + CHECKSERT(res < 0 && errno == EADDRINUSE, "Port already bound (EADDRINUSE)"); + + + INFO("TCP Socket", "listen()"); + + res = listen(fd, 5); + CHECKSERT(res == 0, "Listen OK"); + + INFO("TCP Socket", "accept()"); + int cfd = accept(fd, NULL, NULL); + + CHECKSERT(cfd > 0, "Accepted"); + + unsigned char recvbuf[BUFSIZE]; // recv buffer + + INFO("TCP Socket", "recv()"); + res = recv(cfd, recvbuf, BUFSIZE, MSG_WAITALL); + CHECKSERT(res > 0, "Received data MSG_WAITALL (%i bytes)", res); + recvbuf[res] = 0; + + const char* rm_message = "POSIX is for hipsters"; + CHECKSERT(strcmp((char*)&recvbuf, rm_message) == 0, "Message is \"%s\"", recvbuf); + + memset(recvbuf, 0, BUFSIZE); + res = recv(cfd, recvbuf, BUFSIZE, 0); + recvbuf[res] = 0; + CHECKSERT(res == 0, "No data received (closing)"); + + res = shutdown(cfd, SHUT_RDWR); + + // We cant see if the buffer now is empty without blocking the test + + INFO("TCP Socket", "connect()"); + INFO2("Trigger TCP_recv"); + + cfd = socket(AF_INET, SOCK_STREAM, 0); + // destination address + struct sockaddr_in destaddr; + + memset((char *)&destaddr, 0, sizeof(destaddr)); + destaddr.sin_family = AF_INET; + destaddr.sin_addr.s_addr = htonl(inet.gateway().whole); + destaddr.sin_port = htons(OUT_PORT); + + res = connect(cfd, (struct sockaddr *)&destaddr, sizeof(destaddr)); + CHECKSERT(res == 0, "Connect to remote address OK"); + + INFO("TCP Socket", "send()"); + + const char *my_message = "Only hipsters uses POSIX"; + res = send(cfd, my_message, strlen(my_message), 0); + CHECKSERT(res > 0, "Send works when connected (verified by script)"); + + return 0; +} + +void Service::start(const std::string&) +{ + main(); + Timers::oneshot(std::chrono::seconds(1), [](auto) { + printf("SUCCESS\n"); + }); +} diff --git a/test/posix/integration/tcp/test.py b/test/posix/integration/tcp/test.py new file mode 100755 index 0000000000..1c53a56f4e --- /dev/null +++ b/test/posix/integration/tcp/test.py @@ -0,0 +1,70 @@ +#! /usr/bin/env python + +import sys +import os +import subprocess +import atexit + +includeos_src = os.environ.get('INCLUDEOS_SRC', + os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) +print 'includeos_src: {0}'.format(includeos_src) +sys.path.insert(0,includeos_src) + +from vmrunner import vmrunner +vm = vmrunner.vms[0] + +import socket + +# Set up a temporary interface +import platform +if platform.system() == 'Darwin': + subprocess.call(["sudo", "ifconfig", "bridge43", "alias", "10.0.0.4/24"]) +else: + subprocess.call(["sudo", "ifconfig", "bridge43:2", "10.0.0.4/24"]) + +# Tear down interface on exit +@atexit.register +def tear_down(): + if platform.system() == 'Darwin': + subprocess.call(["sudo", "ifconfig", "bridge43", "-alias", "10.0.0.4"]) + else: + subprocess.call(["sudo", "ifconfig", "bridge43:2", "down"]) + + +S_HOST, S_PORT = '10.0.0.4', 4242 +S_MESSAGE = "Only hipsters uses POSIX" +server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +server.bind((S_HOST, S_PORT)) + +HOST, PORT = '10.0.0.51', 1042 + +RECEIVED = '' + +def verify_recv(recv): + ok = recv == S_MESSAGE + return ok + +def TCP_connect(): + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect((HOST, PORT)) + MESSAGE = "POSIX is for hipsters" + sock.send(MESSAGE) + +def TCP_recv(trigger_line): + server.listen(1) + conn, addr = server.accept() + RECEIVED = conn.recv(1024) + conn.close() + return verify_recv(RECEIVED) + +import thread +def TCP_connect_thread(trigger_line): + thread.start_new_thread(TCP_connect, ()) + +# Add custom event-handler +vm.on_output("accept()", TCP_connect_thread) +vm.on_output("Trigger TCP_recv", TCP_recv) + +# Boot the VM, taking a timeout as parameter +vm.cmake().boot(10).clean() diff --git a/test/posix/integration/tcp/vm.json b/test/posix/integration/tcp/vm.json new file mode 100644 index 0000000000..b805b77520 --- /dev/null +++ b/test/posix/integration/tcp/vm.json @@ -0,0 +1,6 @@ +{ + "image" : "test_posix_tcp.img", + "net" : [{"device" : "virtio", "mac" : "c0:01:0a:00:00:6a"}], + "mem" : 128, + "time_sensitive" : "True" +} From 8a0d31aae704301e01505fb38664ca5b7a4d7944 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 11 Apr 2018 14:08:41 +0200 Subject: [PATCH 165/723] cmake: Auto-detect libgcc/rt on Linux --- cmake/post.service.cmake | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 09b550cb0e..6642871322 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -274,11 +274,6 @@ set(CRTI "${INSTALL_LOC}/${ARCH}/lib/crti.o") target_link_libraries(service ${CRTI}) target_link_libraries(service ${CRT1}) -add_library(libgcc STATIC IMPORTED) -set_target_properties(libgcc PROPERTIES LINKER_LANGUAGE C) -set_target_properties(libgcc PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libcompiler.a) - - add_library(libos STATIC IMPORTED) set_target_properties(libos PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(libos PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libos.a) @@ -333,8 +328,25 @@ set_target_properties(libc PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/l add_library(libpthread STATIC IMPORTED) set_target_properties(libpthread PROPERTIES LINKER_LANGUAGE C) -set_target_properties(libpthread PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libpthread.a) +set_target_properties(libpthread PROPERTIES IMPORTED_LOCATION "${INSTALL_LOC}/${ARCH}/lib/libpthread.a") + +# libgcc/compiler-rt detection +if (UNIX) + execute_process( + COMMAND ${CMAKE_CXX_COMPILER} --target=${TRIPLE} --print-libgcc-file-name + RESULT_VARIABLE CC_RT_RES + OUTPUT_VARIABLE COMPILER_RT_FILE OUTPUT_STRIP_TRAILING_WHITESPACE) + if (NOT ${CC_RT_RES} EQUAL 0) + message(AUTHOR_WARNING "Failed to detect libgcc/compiler-rt: ${COMPILER_RT_FILE}") + endif() +endif() +if (NOT COMPILER_RT_FILE) + set(COMPILER_RT_FILE "${INSTALL_LOC}/${ARCH}/lib/libcompiler.a") +endif() +add_library(libgcc STATIC IMPORTED) +set_target_properties(libgcc PROPERTIES LINKER_LANGUAGE C) +set_target_properties(libgcc PROPERTIES IMPORTED_LOCATION "${COMPILER_RT_FILE}") if ("${PLATFORM}" STREQUAL "x86_solo5") add_library(solo5 STATIC IMPORTED) @@ -423,11 +435,6 @@ if(TARFILE) target_link_libraries(service --whole-archive tarfile --no-whole-archive) endif(TARFILE) -#add_library(crtn STATIC IMPORTED) -#set_target_properties(crtn PROPERTIES LINKER_LANGUAGE CXX) -#set_target_properties(crtn PROPERTIES IMPORTED_LOCATION ${INSTALL_LOC}/${ARCH}/lib/libcrtn.a) - - if ("${PLATFORM}" STREQUAL "x86_solo5") target_link_libraries(service solo5) endif() From 1812b184e4bb51ce8a7248453f7b1a812acf9470 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 11 Apr 2018 21:47:37 +0200 Subject: [PATCH 166/723] cmake: Update botan bundles to musl variants --- cmake/botan.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/botan.cmake b/cmake/botan.cmake index 8c2ef0fbe4..b5a3581e69 100644 --- a/cmake/botan.cmake +++ b/cmake/botan.cmake @@ -4,14 +4,14 @@ include(ExternalProject) if(${ARCH} STREQUAL "x86_64") - set(BOTAN_HASH 707bb3ce80c8fbd8bf84e15211f68936) + set(BOTAN_HASH b1dfea0d906783e00dc650832d47ffd4) elseif(${ARCH} STREQUAL "i686") - set(BOTAN_HASH 788fbe7e214d22638135c5a603bc0fe7) + set(BOTAN_HASH e49e7177fe015de294acb0c8dc13a099) endif() ExternalProject_Add(botan PREFIX botan - URL https://github.com/includeos/botan/releases/download/inc-3.0/botan-includeos-${ARCH}.tar.gz + URL https://github.com/includeos/botan/releases/download/musl-1.0/botan-includeos-${ARCH}.tar.gz URL_HASH MD5=${BOTAN_HASH} CONFIGURE_COMMAND "" BUILD_COMMAND "" From 428c8db6cfb41cc084f9c7de7b2bd3bed40a1b05 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 11 Apr 2018 21:47:58 +0200 Subject: [PATCH 167/723] musl: Add a few syscall stubs needed by Botan --- src/musl/CMakeLists.txt | 5 ++++- src/musl/kill.cpp | 30 ++++++++++++++++++++++++++++++ src/musl/select.cpp | 22 ++++++++++++++++++++++ src/musl/setrlimit.cpp | 14 ++++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 src/musl/kill.cpp create mode 100644 src/musl/select.cpp create mode 100644 src/musl/setrlimit.cpp diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index af5c90f208..42e1b3df72 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -10,19 +10,22 @@ set(MUSL_OBJECTS msync.cpp mincore.cpp syscall_n.cpp sigmask.cpp gettid.cpp tkill.cpp socketcall.cpp rt_sigaction.cpp stat.cpp fstat.cpp fstatat.cpp + access.cpp chmod.cpp cwd.cpp fchmod.cpp fchmodat.cpp getdents.cpp + kill.cpp mkdir.cpp mkdirat.cpp mknodat.cpp openat.cpp + select.cpp + setrlimit.cpp umask.cpp utimensat.cpp uname.cpp - access.cpp ) add_library(musl_syscalls STATIC ${MUSL_OBJECTS}) diff --git a/src/musl/kill.cpp b/src/musl/kill.cpp new file mode 100644 index 0000000000..5abf4e3d1f --- /dev/null +++ b/src/musl/kill.cpp @@ -0,0 +1,30 @@ +#include "common.hpp" +#include + +int sys_kill(pid_t pid, int sig) { + panic("KILL called"); +} + +int sys_tkill(int tid, int sig) { + panic("TKILL called"); +} + +int sys_tgkill(int tgid, int tid, int sig) { + panic("TGKILL called"); +} + +extern "C" +long syscall_SYS_kill(pid_t pid, int sig) { + return strace(sys_kill, "kill", pid, sig); +} + +extern "C" +long syscall_SYS_tkill(int tid, int sig) { + return strace(sys_tkill, "tkill", tid, sig); +} + +extern "C" +long syscall_SYS_tgkill(int tgid, int tid, int sig) +{ + return strace(sys_tgkill, "tgkill", tgid, tid, sig); +} diff --git a/src/musl/select.cpp b/src/musl/select.cpp new file mode 100644 index 0000000000..c2dfbb05e2 --- /dev/null +++ b/src/musl/select.cpp @@ -0,0 +1,22 @@ +#include "common.hpp" +#include +#include + +long sys_select(int nfds, + fd_set* readfds, + fd_set* writefds, + fd_set* exceptfds, + struct timeval* timeout) +{ + return -ENOSYS; +} + +extern "C" +long syscall_SYS_select(int nfds, + fd_set* readfds, + fd_set* writefds, + fd_set* exceptfds, + struct timeval* timeout) +{ + return strace(sys_select, "select", nfds, readfds, writefds, exceptfds, timeout); +} diff --git a/src/musl/setrlimit.cpp b/src/musl/setrlimit.cpp new file mode 100644 index 0000000000..e51bdd192b --- /dev/null +++ b/src/musl/setrlimit.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include + +long sys_setrlimit(int resource, const struct rlimit *rlim) +{ + return -ENOSYS; +} + +extern "C" +long syscall_SYS_setrlimit(int resource, + const struct rlimit* rlim) +{ + return strace(sys_setrlimit, "setrlimit", resource, rlim); +} From da150b95471095fc1a0b01497cef8a7d042aa5d9 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 11 Apr 2018 21:48:15 +0200 Subject: [PATCH 168/723] cmake: Optimize linking, not as many duplicates --- cmake/post.service.cmake | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 6642871322..4a1807daa5 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -448,26 +448,17 @@ target_link_libraries(service libplatform libarch - libos - libc - musl_syscalls - cxxabi - libpthread - libunwind - libcxx - libplatform - libarch + musl_syscalls libos - libc + libcxx cxxabi - libpthread libunwind - libcxx + libpthread + libc musl_syscalls libos - libcxx libc libgcc ${CRTN} From fac5c91eb44a3ff5ca52c72ebbc652f16c02746d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 11 Apr 2018 22:07:01 +0200 Subject: [PATCH 169/723] cmake: Disable building .img files always --- cmake/post.service.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 4a1807daa5..f5ab771c56 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -481,9 +481,9 @@ if (NOT debug) ) endif() -# create .img files too automatically +# create bare metal .img: make legacy_bootloader add_custom_target( - prepend_bootloader ALL + legacy_bootloader COMMAND ${INSTALL_LOC}/bin/vmbuild ${BINARY} ${INSTALL_LOC}/${ARCH}/boot/bootloader DEPENDS service ) From 3094cc2a569c5d747323410687b85cfd438f888c Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 11 Apr 2018 22:07:48 +0200 Subject: [PATCH 170/723] test: Increase timeout on Tar test --- test/util/integration/tar/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/util/integration/tar/test.py b/test/util/integration/tar/test.py index 332f0fd668..87a869b765 100755 --- a/test/util/integration/tar/test.py +++ b/test/util/integration/tar/test.py @@ -123,4 +123,4 @@ def check_num_outputs(line): vm.on_output("Something special to close with", check_num_outputs) # Boot the VM, taking a timeout as parameter -vm.cmake().boot(20).clean() +vm.cmake().boot(30).clean() From 02a76194bc4e3e153065ffea24dae0c475a11c0d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 11 Apr 2018 23:35:58 +0200 Subject: [PATCH 171/723] musl: Add syscalls needed by SQlite --- src/musl/CMakeLists.txt | 10 +++++++++- src/musl/chown.cpp | 13 +++++++++++++ src/musl/fchown.cpp | 13 +++++++++++++ src/musl/fsync.cpp | 13 +++++++++++++ src/musl/ftruncate.cpp | 14 ++++++++++++++ src/musl/geteuid.cpp | 14 ++++++++++++++ src/musl/readlink.cpp | 14 ++++++++++++++ src/musl/rmdir.cpp | 14 ++++++++++++++ src/musl/unlink.cpp | 14 ++++++++++++++ 9 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 src/musl/chown.cpp create mode 100644 src/musl/fchown.cpp create mode 100644 src/musl/fsync.cpp create mode 100644 src/musl/ftruncate.cpp create mode 100644 src/musl/geteuid.cpp create mode 100644 src/musl/readlink.cpp create mode 100644 src/musl/rmdir.cpp create mode 100644 src/musl/unlink.cpp diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index 42e1b3df72..66a94e647f 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -12,20 +12,28 @@ set(MUSL_OBJECTS stat.cpp fstat.cpp fstatat.cpp access.cpp chmod.cpp + chown.cpp cwd.cpp fchmod.cpp fchmodat.cpp + fchown.cpp + fsync.cpp + ftruncate.cpp getdents.cpp + geteuid.cpp kill.cpp mkdir.cpp mkdirat.cpp mknodat.cpp openat.cpp + readlink.cpp + rmdir.cpp select.cpp setrlimit.cpp umask.cpp - utimensat.cpp uname.cpp + utimensat.cpp + unlink.cpp ) add_library(musl_syscalls STATIC ${MUSL_OBJECTS}) diff --git a/src/musl/chown.cpp b/src/musl/chown.cpp new file mode 100644 index 0000000000..531807b9df --- /dev/null +++ b/src/musl/chown.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" +#include + +static long sys_chown(const char* /*path*/, uid_t /*owner*/, gid_t /*group*/) +{ + return -ENOSYS; +} + +extern "C" +long syscall_SYS_chown(const char *path, uid_t owner, gid_t group) +{ + return strace(sys_chown, "chown", path, owner, group); +} diff --git a/src/musl/fchown.cpp b/src/musl/fchown.cpp new file mode 100644 index 0000000000..3e803e8166 --- /dev/null +++ b/src/musl/fchown.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" +#include + +static long sys_fchown(int /*fd*/, uid_t /*owner*/, gid_t /*group*/) +{ + return -ENOSYS; +} + +extern "C" +long syscall_SYS_fchown(int fd, uid_t owner, gid_t group) +{ + return strace(sys_fchown, "fchown", fd, owner, group); +} diff --git a/src/musl/fsync.cpp b/src/musl/fsync.cpp new file mode 100644 index 0000000000..96698c5de7 --- /dev/null +++ b/src/musl/fsync.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" +#include + +static long sys_fsync(int /*fd*/) +{ + return -EROFS; +} + +extern "C" +long syscall_SYS_fsync(int fd) +{ + return strace(sys_fsync, "fsync", fd); +} diff --git a/src/musl/ftruncate.cpp b/src/musl/ftruncate.cpp new file mode 100644 index 0000000000..96113ffa38 --- /dev/null +++ b/src/musl/ftruncate.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include +#include + +static long sys_ftruncate(int /*fd*/, off_t /*length*/) +{ + return -EROFS; +} + +extern "C" +long syscall_SYS_ftruncate(int fd, off_t length) +{ + return strace(sys_ftruncate, "ftruncate", fd, length); +} diff --git a/src/musl/geteuid.cpp b/src/musl/geteuid.cpp new file mode 100644 index 0000000000..beacbc74ba --- /dev/null +++ b/src/musl/geteuid.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include +#include + +static long sys_geteuid() +{ + return 0; +} + +extern "C" +long syscall_SYS_geteuid() +{ + return strace(sys_geteuid, "geteuid"); +} diff --git a/src/musl/readlink.cpp b/src/musl/readlink.cpp new file mode 100644 index 0000000000..9912810aad --- /dev/null +++ b/src/musl/readlink.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include +#include + +static long sys_readlink(const char* /*path*/, char* /*buf*/, size_t /*bufsiz*/) +{ + return -EIO; +} + +extern "C" +long syscall_SYS_readlink(const char *path, char *buf, size_t bufsiz) +{ + return strace(sys_readlink, "readlink", path, buf, bufsiz); +} diff --git a/src/musl/rmdir.cpp b/src/musl/rmdir.cpp new file mode 100644 index 0000000000..6b7bb9f80a --- /dev/null +++ b/src/musl/rmdir.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include + +static long sys_rmdir(const char* /*pathname*/) +{ + /* technically path needs to be verified first */ + return -EROFS; +} + +extern "C" +long syscall_SYS_rmdir(const char *pathname) +{ + return strace(sys_rmdir, "rmdir", pathname); +} diff --git a/src/musl/unlink.cpp b/src/musl/unlink.cpp new file mode 100644 index 0000000000..3fed0c6116 --- /dev/null +++ b/src/musl/unlink.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include + +static long sys_unlink(const char* /*pathname*/) +{ + /* technically path needs to be verified first */ + return -EROFS; +} + +extern "C" +long syscall_SYS_unlink(const char *pathname) +{ + return strace(sys_unlink, "rmdir", pathname); +} From cd84a9f4a1355ac5244685a141a69bc2eeabac7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 12 Apr 2018 10:08:47 +0200 Subject: [PATCH 172/723] musl: added missing return in getdents --- src/musl/getdents.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/musl/getdents.cpp b/src/musl/getdents.cpp index 484a3ded42..137e613ad9 100644 --- a/src/musl/getdents.cpp +++ b/src/musl/getdents.cpp @@ -5,7 +5,7 @@ static long sys_getdents(unsigned int fd, struct dirent *dirp, unsigned int count) { if(auto* fildes = FD_map::_get(fd); fildes) - fildes->getdents(dirp, count); + return fildes->getdents(dirp, count); return -EBADF; } From 93be44663811c2e81e109fc26458ca3d255daaf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 12 Apr 2018 10:29:15 +0200 Subject: [PATCH 173/723] musl: Only support AF_INET (ip4) sockets --- src/musl/socketcall.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/musl/socketcall.cpp b/src/musl/socketcall.cpp index 9015f66031..cae35a3297 100644 --- a/src/musl/socketcall.cpp +++ b/src/musl/socketcall.cpp @@ -8,8 +8,8 @@ static long sock_socket(int domain, int type, int protocol) { - // disallow strange domains, like ALG - if (UNLIKELY(domain < 0 || domain > AF_INET6)) + // currently only support for AF_INET (IPv4, no local/unix or IP6) + if (UNLIKELY(domain != AF_INET)) return -EAFNOSUPPORT; // disallow RAW etc if (UNLIKELY(type < 0 || type > SOCK_DGRAM)) From 5b198a8f6f6bdf456f76950bf3127a1930551465 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 12 Apr 2018 10:49:00 +0200 Subject: [PATCH 174/723] examples: Add vfs plugin to mender example, as its using /dev/random --- examples/mender/CMakeLists.txt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/mender/CMakeLists.txt b/examples/mender/CMakeLists.txt index a2f843c247..2538fbdcce 100644 --- a/examples/mender/CMakeLists.txt +++ b/examples/mender/CMakeLists.txt @@ -10,16 +10,14 @@ include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) project (mender_demo) set(SERVICE_NAME "IncludeOS Mender Example") - set(BINARY "mender_demo") -# Maximum memory can be hard-coded into the binary -set(MAX_MEM 128) - set(SOURCES service.cpp) set(DRIVERS virtionet vmxnet3) +set(PLUGINS vfs) + set(LIBRARIES "libmender.a" "libliveupdate.a" From e4d88df78f2b067fb345f3d3c58d22b9229af4c7 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 12 Apr 2018 11:40:00 +0200 Subject: [PATCH 175/723] test: Increase timeout on net/gateway test --- test/net/integration/gateway/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net/integration/gateway/test.py b/test/net/integration/gateway/test.py index dd06c8b6e4..f4f19408b6 100755 --- a/test/net/integration/gateway/test.py +++ b/test/net/integration/gateway/test.py @@ -8,4 +8,4 @@ sys.path.insert(0,includeos_src) from vmrunner import vmrunner -vmrunner.vms[0].cmake().boot(20).clean() +vmrunner.vms[0].cmake().boot(40).clean() From aae6b865a01544694523de787c67453200477662 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 12 Apr 2018 12:31:53 +0200 Subject: [PATCH 176/723] kernel: Add RNG for Botan, update mender and https server --- api/kernel/botan_rng.hpp | 69 ++++++++++++++++++++++++++++++++++ examples/mender/CMakeLists.txt | 2 - lib/mender/src/keystore.cpp | 16 ++++---- src/net/https/botan_server.cpp | 6 ++- 4 files changed, 81 insertions(+), 12 deletions(-) create mode 100644 api/kernel/botan_rng.hpp diff --git a/api/kernel/botan_rng.hpp b/api/kernel/botan_rng.hpp new file mode 100644 index 0000000000..97649fdfae --- /dev/null +++ b/api/kernel/botan_rng.hpp @@ -0,0 +1,69 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef KERNEL_BOTAN_RNG_HPP +#define KERNEL_BOTAN_RNG_HPP + +#include +#include +#include +#include + +/** +* An interface to a cryptographic random number generator +*/ +class IncludeOS_RNG : public Botan::RandomNumberGenerator +{ +public: + IncludeOS_RNG() = default; + virtual ~IncludeOS_RNG() = default; + + void randomize(uint8_t output[], size_t length) override + { + rng_extract(&output[0], length); + } + + void add_entropy(const uint8_t input[], size_t length) override + { + rng_absorb(&input[0], length); + this->seed_bytes += length; + } + + std::string name() const override { + return "IncludeOS RNG"; + } + + void clear() override { + this->seed_bytes = 0; + } + + bool is_seeded() const override { + return true; //this->seed_bytes >= 4096; + } + + static Botan::RandomNumberGenerator& get() { + static IncludeOS_RNG rng; + return rng; + } +private: + size_t seed_bytes = 0; + IncludeOS_RNG(const IncludeOS_RNG& rng) = delete; + IncludeOS_RNG& operator=(const IncludeOS_RNG& rng) = delete; +}; + +#endif diff --git a/examples/mender/CMakeLists.txt b/examples/mender/CMakeLists.txt index 2538fbdcce..fb15d890e3 100644 --- a/examples/mender/CMakeLists.txt +++ b/examples/mender/CMakeLists.txt @@ -16,8 +16,6 @@ set(SOURCES service.cpp) set(DRIVERS virtionet vmxnet3) -set(PLUGINS vfs) - set(LIBRARIES "libmender.a" "libliveupdate.a" diff --git a/lib/mender/src/keystore.cpp b/lib/mender/src/keystore.cpp index 9d92f024a6..7cc16ad2d7 100644 --- a/lib/mender/src/keystore.cpp +++ b/lib/mender/src/keystore.cpp @@ -20,10 +20,9 @@ #include #include #include -#include -#include #include #include +#include namespace mender { @@ -48,8 +47,8 @@ namespace mender { { MENDER_INFO("Keystore", "Constructing keystore with the private key itself"); Botan::DataSource_Memory data{key}; - private_key_.reset(dynamic_cast(Botan::PKCS8::load_key(data, Botan::system_rng()))); - //assert(private_key_->check_key(Botan::system_rng(), true)); + private_key_.reset(dynamic_cast(Botan::PKCS8::load_key(data, IncludeOS_RNG::get()))); + //assert(private_key_->check_key(IncludeOS_RNG::get(), true)); } void Keystore::load() @@ -59,7 +58,7 @@ namespace mender { if(buffer) { Botan::DataSource_Memory data{buffer.data(), buffer.size()}; - private_key_.reset(dynamic_cast(Botan::PKCS8::load_key(data, Botan::system_rng()))); + private_key_.reset(dynamic_cast(Botan::PKCS8::load_key(data, IncludeOS_RNG::get()))); MENDER_INFO2("%s\n ... Done.", Botan::PKCS8::PEM_encode(*private_key_).c_str()); } else @@ -77,7 +76,7 @@ namespace mender { void Keystore::generate() { MENDER_INFO("Keystore", "Generating private key ..."); - private_key_ = std::make_unique(Botan::system_rng(), 1024); + private_key_ = std::make_unique(IncludeOS_RNG::get(), 1024); MENDER_INFO2("%s\n ... Done.", Botan::PKCS8::PEM_encode(*private_key_).c_str()); save(); } @@ -93,8 +92,9 @@ namespace mender { byte_seq Keystore::sign(const byte_seq& data) { // https://botan.randombit.net/manual/pubkey.html#signatures - Botan::PK_Signer signer(*private_key_, "EMSA3(SHA-256)"); // EMSA3 for backward compability with PKCS1_15 - auto signature = signer.sign_message((uint8_t*)data.data(), data.size(), Botan::system_rng()); + // EMSA3 for backward compability with PKCS1_15 + Botan::PK_Signer signer(*private_key_, IncludeOS_RNG::get(), "EMSA3(SHA-256)"); + auto signature = signer.sign_message((uint8_t*)data.data(), data.size(), IncludeOS_RNG::get()); //printf("Sign:\n%s\n", Botan::hex_encode(signature).c_str()); return signature; } diff --git a/src/net/https/botan_server.cpp b/src/net/https/botan_server.cpp index 745bc4fbf9..6c31a997a3 100644 --- a/src/net/https/botan_server.cpp +++ b/src/net/https/botan_server.cpp @@ -16,11 +16,13 @@ // limitations under the License. #include +#include #include -#include #include -inline static auto& get_rng() { return Botan::system_rng(); } +inline static Botan::RandomNumberGenerator& get_rng() { + return IncludeOS_RNG::get(); +} inline std::unique_ptr read_pkey(fs::Dirent& key_file) { From 85fe19f04a94823b6df55ff77cd25d9ae1dca265 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 12 Apr 2018 16:29:09 +0200 Subject: [PATCH 177/723] alloc_lstack: add free / used functionality --- api/util/alloc_lstack.hpp | 72 ++++++++++++++++++++++++--------------- test/util/unit/lstack.cpp | 10 +++++- 2 files changed, 54 insertions(+), 28 deletions(-) diff --git a/api/util/alloc_lstack.hpp b/api/util/alloc_lstack.hpp index dc92642c01..896e743b2f 100644 --- a/api/util/alloc_lstack.hpp +++ b/api/util/alloc_lstack.hpp @@ -62,7 +62,49 @@ class Lstack { void* allocate(size_t size){ Expects((size & (Min - 1)) == 0); + auto* chunk = find_free(size); + if (chunk != nullptr) + bytes_allocated_ += chunk->size; + return chunk; + } + + void deallocate (void* ptr, size_t size){ + push(ptr, size); + bytes_allocated_ -= size; + } + + void donate(void* ptr, size_t size){ + push(ptr, size); + bytes_total_ += size; + } + + const Chunk* begin() const { + return front_; + } + + bool empty() const noexcept { return front_ == nullptr || front_->size == 0; } + + size_t bytes_allocated() + { return bytes_allocated_; } + + size_t bytes_free() + { return bytes_total_ - bytes_allocated_; } + +private: + Chunk* new_chunk(void* addr_begin, Chunk* next, size_t sz){ + Expects((sz & align - 1) == 0); + Expects(((uintptr_t)addr_begin & align - 1) == 0); + return new ((Chunk*)addr_begin) Chunk(next, sz); + } + + void push(void* ptr, size_t size){ + Expects((size & (align - 1)) == 0); + auto* old_front = front_; + front_ = (Chunk*)ptr; + new_chunk(front_, old_front, size); + }; + Chunk* find_free(size_t size) { Chunk* next = front_; Chunk** prev = &front_; do { @@ -84,6 +126,7 @@ class Lstack { *prev = nullptr; else *prev = new_chunk((char*)next + size, next->next, next->size - size); + next->size = size; return next; } @@ -94,34 +137,9 @@ class Lstack { return nullptr; } - void deallocate (void* ptr, size_t size){ - push(ptr, size); - } - - void donate(void* ptr, size_t size){ - push(ptr, size); - } - - void push(void* ptr, size_t size){ - Expects((size & (align - 1)) == 0); - auto* old_front = front_; - front_ = (Chunk*)ptr; - new_chunk(front_, old_front, size); - }; - - const Chunk* begin() const { - return front_; - } - - bool empty() const noexcept { return front_ == nullptr || front_->size == 0; } - -private: - Chunk* new_chunk(void* addr_begin, Chunk* next, size_t sz){ - Expects((sz & align - 1) == 0); - Expects(((uintptr_t)addr_begin & align - 1) == 0); - return new ((Chunk*)addr_begin) Chunk(next, sz); - } Chunk* front_ = nullptr; + ssize_t bytes_allocated_ = 0; + ssize_t bytes_total_ = 0; }; } // namespace util diff --git a/test/util/unit/lstack.cpp b/test/util/unit/lstack.cpp index 8fbae527dd..13c5dbeb08 100644 --- a/test/util/unit/lstack.cpp +++ b/test/util/unit/lstack.cpp @@ -64,19 +64,27 @@ CASE("Using lstack") char* pool = (char*)memalign(blocksize, poolsize); void* pool_end = pool + poolsize; heap.donate(pool, poolsize); + EXPECT(heap.bytes_allocated() == 0); + EXPECT(heap.bytes_free() == poolsize); auto* page = heap.allocate(blocksize); EXPECT(page == pool); - EXPECT(((Chunk*)page)->size == poolsize); + EXPECT(((Chunk*)page)->size == blocksize); EXPECT(((Chunk*)page)->next == nullptr); + EXPECT(heap.bytes_allocated() == blocksize); + EXPECT(heap.bytes_free() == poolsize - blocksize); print_summary(heap.begin()); int i = 0; + auto prev_bytes_alloc = heap.bytes_allocated(); for (; i < poolsize - blocksize; i += blocksize) { auto* p = heap.allocate(blocksize); EXPECT(p == (uint8_t*)pool + blocksize + i); + EXPECT(heap.bytes_allocated() == prev_bytes_alloc + blocksize); + prev_bytes_alloc = heap.bytes_allocated(); + EXPECT(heap.bytes_free() == poolsize - prev_bytes_alloc); } print_summary(heap.begin()); From 4c64b47a2aaf4f5530db2d835468a8b6e35bc4d2 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 12 Apr 2018 17:30:01 +0200 Subject: [PATCH 178/723] kernel: integrate heap usage and heap initialization with brk/mmap --- api/arch/i686.hpp | 2 ++ api/arch/x86_64.hpp | 2 ++ api/kernel/os.hpp | 11 +++++-- src/crt/c_abi.c | 1 - src/kernel/heap.cpp | 44 +++++++++++++++++++++---- src/kernel/multiboot.cpp | 22 +++++++++---- src/kernel/os.cpp | 5 +-- src/kernel/syscalls.cpp | 6 ++-- src/musl/brk.cpp | 44 +++++++++++++------------ src/musl/mmap.cpp | 16 ++++++--- src/platform/x86_nano/kernel_start.cpp | 10 ++++-- src/platform/x86_pc/kernel_start.cpp | 12 ++++--- src/platform/x86_pc/os.cpp | 14 ++++---- src/platform/x86_solo5/kernel_start.cpp | 2 +- src/platform/x86_solo5/os.cpp | 12 ++----- 15 files changed, 133 insertions(+), 70 deletions(-) diff --git a/api/arch/i686.hpp b/api/arch/i686.hpp index 4a44ff5c0a..de0098a991 100644 --- a/api/arch/i686.hpp +++ b/api/arch/i686.hpp @@ -28,4 +28,6 @@ inline uint64_t __arch_cpu_cycles() noexcept { } +constexpr uintptr_t __arch_max_canonical_addr = 0xffffffff; + #endif diff --git a/api/arch/x86_64.hpp b/api/arch/x86_64.hpp index 6a7cb8d4df..75e7958c91 100644 --- a/api/arch/x86_64.hpp +++ b/api/arch/x86_64.hpp @@ -27,4 +27,6 @@ inline uint64_t __arch_cpu_cycles() noexcept { return ((uint64_t) lo) | ((uint64_t) hi) << 32; } +constexpr uintptr_t __arch_max_canonical_addr = 0xffffffffffff; + #endif diff --git a/api/kernel/os.hpp b/api/kernel/os.hpp index fcccce9690..fec5c7efdb 100644 --- a/api/kernel/os.hpp +++ b/api/kernel/os.hpp @@ -175,7 +175,10 @@ class OS { } /** Total used dynamic memory, in bytes */ - static uintptr_t heap_usage() noexcept; + static size_t heap_usage() noexcept; + + /** Total free heap, as far as the OS knows, in bytes */ + static size_t heap_avail() noexcept; /** Attempt to trim the heap end, reducing the size */ static void heap_trim() noexcept; @@ -197,6 +200,8 @@ class OS { return memory_end_; } + static void init_heap(uintptr_t phys_begin, size_t size) noexcept; + /** * Returns true when the current OS comes from a live update, * as opposed to booting from either a rollback or a normal boot @@ -271,8 +276,10 @@ class OS { static uintptr_t liveupdate_loc_; static std::string version_str_; static std::string arch_str_; - static uintptr_t memory_end_; + static uintptr_t heap_begin_; + static uintptr_t heap_end_; static uintptr_t heap_max_; + static uintptr_t memory_end_; static const uintptr_t elf_binary_size_; static const char* cmdline; static Panic_action panic_action_; diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index 99d164b734..21a59c2770 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -22,7 +22,6 @@ #define HEAP_ALIGNMENT 63 void* __dso_handle; -uintptr_t heap_end; uint32_t _move_symbols(void* sym_loc) { diff --git a/src/kernel/heap.cpp b/src/kernel/heap.cpp index 2c65ab5a92..47e22c3987 100644 --- a/src/kernel/heap.cpp +++ b/src/kernel/heap.cpp @@ -16,12 +16,25 @@ // limitations under the License. #include -extern uintptr_t heap_begin; -extern uintptr_t heap_end; +#include -uintptr_t OS::heap_usage() noexcept +size_t brk_bytes_used(); +size_t mmap_bytes_used(); + +uintptr_t OS::heap_begin_ = 0; +uintptr_t OS::heap_end_ = __arch_max_canonical_addr; +uintptr_t OS::heap_max_ = __arch_max_canonical_addr; +uintptr_t OS::memory_end_ = __arch_max_canonical_addr;; + + +size_t OS::heap_usage() noexcept +{ + return brk_bytes_used() + mmap_bytes_used(); +} + +size_t OS::heap_avail() noexcept { - return ::heap_end; + return (heap_max() - heap_begin()) - heap_usage(); } void OS::heap_trim() noexcept @@ -36,11 +49,11 @@ uintptr_t OS::heap_max() noexcept uintptr_t OS::heap_begin() noexcept { - return ::heap_begin; + return heap_begin_; } uintptr_t OS::heap_end() noexcept { - return ::heap_end; + return heap_end_; } uintptr_t OS::resize_heap(size_t size) @@ -53,3 +66,22 @@ uintptr_t OS::resize_heap(size_t size) heap_max_ = heap_begin() + size; return size; } + +constexpr size_t heap_alignment = 4096; +__attribute__((weak)) ssize_t __brk_max = 0x100000; + +extern void init_mmap(uintptr_t mmap_begin); + + +uintptr_t __init_brk(uintptr_t begin); +uintptr_t __init_mmap(uintptr_t begin); + +void OS::init_heap(uintptr_t free_mem_begin, uintptr_t memory_end) noexcept { + // NOTE: Initialize the heap before exceptions + // cache-align heap, because its not aligned + memory_end_ = memory_end; + heap_max_ = memory_end; + heap_begin_ = util::bits::roundto(free_mem_begin); + auto brk_end = __init_brk(heap_begin_); + heap_end_ = __init_mmap(brk_end); +} diff --git a/src/kernel/multiboot.cpp b/src/kernel/multiboot.cpp index b39277db1e..09ef88bf60 100644 --- a/src/kernel/multiboot.cpp +++ b/src/kernel/multiboot.cpp @@ -21,17 +21,21 @@ #include #include -//#define DEBUG -#if defined(DEBUG) +// #define DEBUG_MULTIBOOT +#if defined(DEBUG_MULTIBOOT) +#undef debug #define debug(X,...) kprintf(X,##__VA_ARGS__); +#define MYINFO(X,...) kprintf("" X "\n", ##__VA_ARGS__) +#undef INFO2 +#define INFO2(X,...) kprintf("\t" X "\n", ##__VA_ARGS__) #else #define debug(X,...) +#define MYINFO(X,...) INFO("Kernel", X, ##__VA_ARGS__) #endif -#define MYINFO(X,...) INFO("Kernel", X, ##__VA_ARGS__) extern uintptr_t _end; -extern "C" uintptr_t _multiboot_free_begin(uintptr_t); + using namespace util::bitops; using namespace util::literals; @@ -48,6 +52,14 @@ multiboot_info_t* OS::bootinfo() return (multiboot_info_t*) (uintptr_t) __multiboot_addr; } +uintptr_t _multiboot_memory_end(uintptr_t boot_addr) { + auto* info = bootinfo(boot_addr); + if (info->flags & MULTIBOOT_INFO_MEMORY) { + return 0x100000 + (info->mem_upper * 1024); + } + return __arch_max_canonical_addr; +} + // Deterimine the end of multiboot provided data // (e.g. multiboot's data area as offset to the _end symbol) uintptr_t _multiboot_free_begin(uintptr_t boot_addr) @@ -109,8 +121,6 @@ void OS::multiboot(uint32_t boot_addr) uint32_t mem_high_end = mem_high_start + (info->mem_upper * 1024) - 1; uint32_t mem_high_kb = info->mem_upper; - OS::memory_end_ = mem_high_end; - INFO2("* Valid memory (%i Kib):", mem_low_kb + mem_high_kb); INFO2(" 0x%08x - 0x%08x (%i Kib)", mem_low_start, mem_low_end, mem_low_kb); diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 5d5a952ad0..4b1907529f 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -41,8 +41,6 @@ using namespace util; extern "C" void* get_cpu_esp(); -extern uintptr_t heap_begin; -extern uintptr_t heap_end; extern uintptr_t _start; extern uintptr_t _end; extern uintptr_t _ELF_START_; @@ -57,8 +55,7 @@ bool OS::m_is_live_updated = false; bool OS::m_block_drivers_ready = false; KHz OS::cpu_khz_ {-1}; uintptr_t OS::liveupdate_loc_ = 0; -uintptr_t OS::memory_end_ = 0; -uintptr_t OS::heap_max_ = (uintptr_t)0xffffffffffff; + OS::Panic_action OS::panic_action_ = OS::Panic_action::PANIC_ACTION; const uintptr_t OS::elf_binary_size_ {(uintptr_t)&_ELF_END_ - (uintptr_t)&_ELF_START_}; diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index 69391a2680..b48fd736b9 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -134,11 +134,11 @@ void panic(const char* why) // heap info typedef unsigned long ulong; - uintptr_t heap_total = OS::heap_max() - heap_begin; + uintptr_t heap_total = OS::heap_max() - OS::heap_begin(); fprintf(stderr, "Heap is at: %p / %p (diff=%lu)\n", - (void*) heap_end, (void*) OS::heap_max(), (ulong) (OS::heap_max() - heap_end)); + (void*) OS::heap_end(), (void*) OS::heap_max(), (ulong) (OS::heap_max() - OS::heap_end())); fprintf(stderr, "Heap usage: %lu / %lu Kb\n", // (%.2f%%)\n", - (ulong) (heap_end - heap_begin) / 1024, + (ulong) (OS::heap_end() - OS::heap_begin()) / 1024, (ulong) heap_total / 1024); //, total * 100.0); print_backtrace(); diff --git a/src/musl/brk.cpp b/src/musl/brk.cpp index d83b8804fb..8b36126e68 100644 --- a/src/musl/brk.cpp +++ b/src/musl/brk.cpp @@ -4,42 +4,44 @@ #include #include -__attribute__((weak)) uintptr_t __brk_max = 0x100000; -uintptr_t heap_begin = 0; -uintptr_t brk_end; -uintptr_t brk_init = 0; +static uintptr_t brk_begin = 0; +static uintptr_t brk_end = 0; +static uintptr_t brk_initialized = 0; +extern ssize_t __brk_max; -extern void init_mmap(uintptr_t mmap_begin); -extern "C" -void _init_heap(uintptr_t free_mem_begin) +uintptr_t __init_brk(uintptr_t begin) { - #define HEAP_ALIGNMENT 4095 - // NOTE: Initialize the heap before exceptions - // cache-align heap, because its not aligned - heap_begin = free_mem_begin + HEAP_ALIGNMENT; - heap_begin = ((uintptr_t)heap_begin & ~HEAP_ALIGNMENT); - brk_end = heap_begin; - brk_init = brk_end; + brk_begin = begin; + brk_end = begin; + brk_initialized = brk_end; kprintf("* Brk initialized. Begin: %p, end %p, MAX %p\n", - (void*) heap_begin, (void*) brk_end, (void*) __brk_max); + (void*) brk_begin, (void*) brk_end, (void*) __brk_max); + return brk_begin + __brk_max; +} + + +size_t brk_bytes_used() { + return brk_end - brk_begin; +} - init_mmap(heap_begin + __brk_max); +size_t brk_bytes_free() { + return __brk_max - brk_bytes_free(); } static uintptr_t sys_brk(void* addr) { if (addr == nullptr - or (uintptr_t)addr > heap_begin + __brk_max - or (uintptr_t)addr < heap_begin) { + or (uintptr_t)addr > brk_begin + __brk_max + or (uintptr_t)addr < brk_begin) { return brk_end; } brk_end = (uintptr_t)addr; - if (brk_end > brk_init) { - memset((void*)brk_init, 0, brk_end - brk_init); - brk_init = brk_end; + if (brk_end > brk_initialized) { + memset((void*)brk_initialized, 0, brk_end - brk_initialized); + brk_initialized = brk_end; } return brk_end; diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 0fa68a4338..84e4fbd907 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -6,18 +6,18 @@ #include #include -extern uintptr_t heap_begin; -extern uintptr_t heap_end; - using Alloc = util::alloc::Lstack<4096>; static Alloc alloc; -void init_mmap(uintptr_t addr_begin) +uintptr_t __init_mmap(uintptr_t addr_begin) { Expects(alloc.empty()); auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); int64_t len = (OS::heap_max() - aligned_begin) & ~int64_t(Alloc::align - 1); alloc.donate((void*)aligned_begin, len); + kprintf("* mmap initialized. Begin: 0x%zx, end: 0x%zx\n", + addr_begin, addr_begin + len); + return aligned_begin + len; } extern "C" __attribute__((weak)) @@ -30,6 +30,14 @@ void kfree (void* ptr, size_t size) { alloc.deallocate(ptr, size); } +size_t mmap_bytes_used() { + return alloc.bytes_allocated(); +} + +size_t mmap_bytes_free() { + return alloc.bytes_free(); +} + static void* sys_mmap(void *addr, size_t length, int /*prot*/, int /*flags*/, int fd, off_t /*offset*/) { diff --git a/src/platform/x86_nano/kernel_start.cpp b/src/platform/x86_nano/kernel_start.cpp index 775838f816..905a8aab3c 100644 --- a/src/platform/x86_nano/kernel_start.cpp +++ b/src/platform/x86_nano/kernel_start.cpp @@ -24,12 +24,16 @@ extern "C" { void __init_serial1(); void __init_sanity_checks(); - uintptr_t _multiboot_free_begin(uintptr_t boot_addr); uintptr_t _move_symbols(uintptr_t loc); void _init_bss(); void _init_heap(uintptr_t); void _init_syscalls(); } + +uintptr_t _multiboot_free_begin(uintptr_t boot_addr); +uintptr_t _multiboot_memory_end(uintptr_t boot_addr); + + bool __libc_initialized = true; extern bool os_default_stdout; @@ -42,9 +46,11 @@ void kernel_start(uintptr_t magic, uintptr_t addr) // Determine where free memory starts extern char _end; uintptr_t free_mem_begin = (uintptr_t) &_end; + uintptr_t mem_end = __arch_max_canonical_addr; if (magic == MULTIBOOT_BOOTLOADER_MAGIC) { free_mem_begin = _multiboot_free_begin(addr); + mem_end = _multiboot_memory_end(addr); } // Preserve symbols from the ELF binary @@ -55,7 +61,7 @@ void kernel_start(uintptr_t magic, uintptr_t addr) __builtin_memset(&_BSS_START_, 0, &_BSS_END_ - &_BSS_START_); // Initialize heap - _init_heap(free_mem_begin); + OS::init_heap(free_mem_begin, mem_end); // Initialize system calls _init_syscalls(); diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 877fbb9104..7c6025ce90 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -35,13 +35,14 @@ extern "C" { void __init_sanity_checks(); void kernel_sanity_checks(); void _init_bss(); - uintptr_t _multiboot_free_begin(uintptr_t boot_addr); uintptr_t _move_symbols(uintptr_t loc); - void _init_heap(uintptr_t free_mem_begin); void _init_elf_parser(); void _init_syscalls(); } +uintptr_t _multiboot_free_begin(uintptr_t bootinfo); +uintptr_t _multiboot_memory_end(uintptr_t bootinfo); + extern char _ELF_START_; extern char _ELF_END_; extern char _INIT_START_; @@ -125,10 +126,13 @@ void kernel_start(uint32_t magic, uint32_t addr) // Determine where free memory starts extern char _end; uintptr_t free_mem_begin = reinterpret_cast(&_end); + uintptr_t memory_end = __arch_max_canonical_addr; if (magic == MULTIBOOT_BOOTLOADER_MAGIC) { free_mem_begin = _multiboot_free_begin(addr); - kprintf("* Free mem begin: 0x%lx \n", free_mem_begin); + memory_end = _multiboot_memory_end(addr); + kprintf("* Free mem begin: 0x%zx, memory end: 0x%zx \n", + free_mem_begin, memory_end); } kprintf("* Moving symbols. \n"); @@ -143,7 +147,7 @@ void kernel_start(uint32_t magic, uint32_t addr) _init_bss(); kprintf("* Init heap\n"); - _init_heap(free_mem_begin); + OS::init_heap(free_mem_begin, memory_end); kprintf("* Init syscalls\n"); _init_syscalls(); diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 7f87962daf..fa14af3584 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -38,8 +38,6 @@ extern "C" void* get_cpu_esp(); //extern "C" void __libc_init_array(); -extern uintptr_t heap_begin; -extern uintptr_t heap_end; extern uintptr_t _start; extern uintptr_t _end; extern uintptr_t _ELF_START_; @@ -109,7 +107,6 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) // BOOT METHOD // PROFILE("Multiboot / legacy"); - OS::memory_end_ = 0; // Detect memory limits etc. depending on boot type if (boot_magic == MULTIBOOT_BOOTLOADER_MAGIC) { OS::multiboot(boot_addr); @@ -120,7 +117,10 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) OS::legacy_boot(); } - assert(OS::memory_end_ != 0); + assert(OS::memory_end() != 0); + + MYINFO("Total memory detected as %s ", util::Byte_r(OS::memory_end_).to_string().c_str()); + // Give the rest of physical memory to heap OS::heap_max_ = OS::memory_end_; @@ -146,16 +146,16 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) memmap.assign_range({0x10000, 0x9d3ff, "Stack"}); #endif - assert(::heap_begin != 0x0 and OS::heap_max_ != 0x0); + assert(heap_begin() != 0x0 and OS::heap_max_ != 0x0); // @note for security we don't want to expose this - memmap.assign_range({(uintptr_t)&_end, ::heap_begin - 1, + memmap.assign_range({(uintptr_t)&_end, heap_begin() - 1, "Pre-heap"}); uintptr_t span_max = std::numeric_limits::max(); uintptr_t heap_range_max_ = std::min(span_max, OS::heap_max_); MYINFO("Assigning heap"); - memmap.assign_range({::heap_begin, heap_range_max_, + memmap.assign_range({heap_begin(), heap_range_max_, "Dynamic memory", heap_usage }); MYINFO("Virtual memory map"); diff --git a/src/platform/x86_solo5/kernel_start.cpp b/src/platform/x86_solo5/kernel_start.cpp index 09d6d5b224..3c1b87c22d 100644 --- a/src/platform/x86_solo5/kernel_start.cpp +++ b/src/platform/x86_solo5/kernel_start.cpp @@ -46,7 +46,7 @@ extern "C" { // XXX: this is dangerous as solo5 might be doing malloc()'s using it's own // idea of a heap. Luckily there is no malloc instance at solo5/kernel/[ukvm|virtio|muen], // so might be OK (for now). - _init_heap(free_mem_begin); + OS::init_heap(free_mem_begin, mem_size); _init_elf_parser(); diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index 8ad31cd4bd..9eee9051ab 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -17,8 +17,6 @@ extern "C" void* get_cpu_esp(); extern char __init_array_start; extern char __init_array_end; int __run_ctors(uintptr_t begin, uintptr_t end); -extern uintptr_t heap_begin; -extern uintptr_t heap_end; extern uintptr_t _start; extern uintptr_t _end; extern uintptr_t mem_size; @@ -91,10 +89,6 @@ void OS::start(char* _cmdline, uintptr_t mem_size) OS::cmdline = _cmdline; - // setup memory and heap end - OS::memory_end_ = mem_size; - OS::heap_max_ = OS::memory_end_; - // Call global ctors PROFILE("Global kernel constructors"); __run_ctors((uintptr_t)&__init_array_start, (uintptr_t)&__init_array_end); @@ -110,16 +104,16 @@ void OS::start(char* _cmdline, uintptr_t mem_size) memmap.assign_range({(uintptr_t)&_LOAD_START_, (uintptr_t)&_end, "ELF"}); - Expects(::heap_begin and heap_max_); + Expects(heap_begin() and heap_max_); // @note for security we don't want to expose this - memmap.assign_range({(uintptr_t)&_end + 1, ::heap_begin - 1, + memmap.assign_range({(uintptr_t)&_end + 1, heap_begin() - 1, "Pre-heap"}); uintptr_t span_max = std::numeric_limits::max(); uintptr_t heap_range_max_ = std::min(span_max, heap_max_); MYINFO("Assigning heap"); - memmap.assign_range({::heap_begin, heap_range_max_, + memmap.assign_range({heap_begin(), heap_range_max_, "Dynamic memory", heap_usage }); MYINFO("Printing memory map"); From e65a77abab1f525d0d737cd142b403a14f95e911 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 12 Apr 2018 17:31:42 +0200 Subject: [PATCH 179/723] test: stress test explicitly fails on 0 memory usage --- test/stress/service.cpp | 2 +- test/stress/test.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/test/stress/service.cpp b/test/stress/service.cpp index a635d54f52..23b1bd74c5 100644 --- a/test/stress/service.cpp +++ b/test/stress/service.cpp @@ -182,7 +182,7 @@ void Service::start(const std::string&) printf("*** TEST SERVICE STARTED *** \n"); auto memuse = OS::heap_usage(); - printf("Current memory usage: %i b, (%f MB) \n", memuse, float(memuse) / 1000000); + printf("Current memory usage: %zi b, (%f MB) \n", memuse, float(memuse) / 1000000); /** These printouts are event-triggers for the vmrunner **/ printf("Ready to start\n"); diff --git a/test/stress/test.py b/test/stress/test.py index 9cab9a8388..6d6d559513 100755 --- a/test/stress/test.py +++ b/test/stress/test.py @@ -56,7 +56,9 @@ def get_mem(): def get_mem_start(): global memuse_at_start - memuse_at_start = get_mem() + if memuse_at_start == 0: + memuse_at_start = get_mem() + return memuse_at_start def memory_increase(lead_time, expected_memuse = memuse_at_start): name_tag = "<" + test_name + "::memory_increase>" @@ -127,7 +129,10 @@ def ARP_burst(burst_size = BURST_SIZE, burst_interval = BURST_INTERVAL): def crash_test(string): print color.INFO("Opening persistent TCP connection for diagnostics") sock_mem.connect((HOST, PORT_MEM)) - get_mem_start() + mem_before = get_mem_start() + if mem_before <= 0: + print color.FAIL("Initial memory reported as " + str(mem_before)) + return False print color.HEADER("Initial crash test") burst_size = BURST_SIZE * 10 @@ -137,7 +142,9 @@ def crash_test(string): ICMP_flood(burst_size, 0) httperf(burst_size, 0) time.sleep(BURST_INTERVAL) - return get_mem() + mem_after = get_mem() + print color.INFO("Crash test complete. Memory in use: "), mem_after + return mem_after >= memuse_at_start # Fire several bursts, e.g. trigger a function that fires bursts, several times def fire_bursts(func, sub_test_name, lead_out = 3): @@ -251,6 +258,6 @@ def wait_for_tw(): BURST_SIZE = int(sys.argv[3]) print color.HEADER(test_name + " initializing") -print color.INFO(name_tag),"Doing", BURST_COUNT,"bursts of", BURST_SIZE, "packets each" +print color.INFO(name_tag),"configured for ", BURST_COUNT,"bursts of", BURST_SIZE, "packets each" vm.cmake().boot(timeout).clean() From a122030cf42249521e986249440186be243bfdf0 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 13 Apr 2018 10:54:15 +0200 Subject: [PATCH 180/723] test/kernel: fix heap_max off-by-one, improve stress test --- src/platform/x86_pc/os.cpp | 6 +++--- test/stress/service.cpp | 41 ++++++++++++++++++++++++++++++++++++++ test/stress/test.py | 14 +++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index fa14af3584..8eafe53e6a 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -122,7 +122,7 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) MYINFO("Total memory detected as %s ", util::Byte_r(OS::memory_end_).to_string().c_str()); // Give the rest of physical memory to heap - OS::heap_max_ = OS::memory_end_; + OS::heap_max_ = OS::memory_end_ - 1; /// STATMAN /// PROFILE("Statman"); @@ -154,8 +154,8 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) uintptr_t span_max = std::numeric_limits::max(); uintptr_t heap_range_max_ = std::min(span_max, OS::heap_max_); - MYINFO("Assigning heap"); - memmap.assign_range({heap_begin(), heap_range_max_, + MYINFO("Assigning heap 0x%zx -> 0x%zx", heap_begin_, heap_range_max_); + memmap.assign_range({heap_begin_, heap_range_max_, "Dynamic memory", heap_usage }); MYINFO("Virtual memory map"); diff --git a/test/stress/service.cpp b/test/stress/service.cpp index 23b1bd74c5..026f1d45a7 100644 --- a/test/stress/service.cpp +++ b/test/stress/service.cpp @@ -71,6 +71,47 @@ uint64_t TCP_BYTES_SENT = 0; void Service::start(const std::string&) { + using namespace util::literals; + // Allocation / free spam to warm up + auto initial_memuse = OS::heap_usage(); + printf("Current memory usage: %zi b, (%s) \n", initial_memuse, util::Byte_r(initial_memuse).to_string().c_str()); + + + std::array allocs {}; + auto chunksize = 1024 * 1024 * 5; + printf("Exercising heap: incrementally allocating %zi x %i bytes \n", + allocs.size(), chunksize); + + for (auto& ptr : allocs) { + ptr = malloc(chunksize); + Expects(ptr != nullptr); + auto memuse = OS::heap_usage(); + printf("Current memory usage: %zi b, (%s) \n", memuse, util::Byte_r(memuse).to_string().c_str()); + Expects(memuse > initial_memuse); + } + + auto high_memuse = OS::heap_usage(); + Expects(high_memuse >= (chunksize * allocs.size()) + initial_memuse); + + auto prev_memuse = high_memuse; + + printf("Deallocating \n"); + for (auto& ptr : allocs) { + free(ptr); + auto memuse = OS::heap_usage(); + printf("Current memory usage: %zi b, (%s) \n", memuse, util::Byte_r(memuse).to_string().c_str()); + Expects(memuse < high_memuse); + Expects(memuse < prev_memuse); + prev_memuse = memuse; + } + + // Expect the heap to have shrunk. With musl and chunksize of 5Mib, + // the mallocs will have resulted in calls to mmap, and frees in calls to + // munmap, so we could expect to be back to exacty where we were, but + // we're adding some room (somewhat arbitrarily) for malloc to change and + // not necessarily give back all it allocated. + Expects(OS::heap_usage() <= initial_memuse + 1_MiB); + printf("Heap functioning as expected\n"); // Timer spam for (int i = 0; i < 1000; i++) diff --git a/test/stress/test.py b/test/stress/test.py index 6d6d559513..6f09426a38 100755 --- a/test/stress/test.py +++ b/test/stress/test.py @@ -38,6 +38,8 @@ sock_mem = socket.socket sock_mem = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +heap_verified = False + def get_mem(): name_tag = "<" + test_name + "::get_mem>" @@ -126,6 +128,13 @@ def ARP_burst(burst_size = BURST_SIZE, burst_interval = BURST_INTERVAL): return get_mem() + +def heap_ok(line): + global heap_verified + heap_verified = True + print color.INFO("Stresstest::heap_ok"), "VM reports heap is increasing and decreasing as expected" + + def crash_test(string): print color.INFO("Opening persistent TCP connection for diagnostics") sock_mem.connect((HOST, PORT_MEM)) @@ -134,6 +143,10 @@ def crash_test(string): print color.FAIL("Initial memory reported as " + str(mem_before)) return False + if not heap_verified: + print color.FAIL("Heap behavior was not verified as expected. ") + return False + print color.HEADER("Initial crash test") burst_size = BURST_SIZE * 10 @@ -240,6 +253,7 @@ def wait_for_tw(): time.sleep(7) # Add custom event-handlers +vm.on_output("Heap functioning as expected", heap_ok) vm.on_output("Ready to start", crash_test) vm.on_output("Ready for ARP", ARP) vm.on_output("Ready for UDP", UDP) From d9b887869b1c1377823cc4907e6790a37cd023d5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 13 Apr 2018 11:18:33 +0200 Subject: [PATCH 181/723] musl: Implement clock_gettime and gettimeofday --- seed/service/service.cpp | 20 +++++++++++++++++++- src/musl/clock_gettime.cpp | 22 +++++++++++++++++++--- src/musl/gettimeofday.cpp | 18 +++++++++++++++--- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/seed/service/service.cpp b/seed/service/service.cpp index eff95fd30b..4d197ac9a5 100644 --- a/seed/service/service.cpp +++ b/seed/service/service.cpp @@ -17,10 +17,28 @@ #include #include +#include +#include + +static void print_time() +{ + /* Obtain the time of day, and convert it to a tm struct. */ + struct timeval tv; + gettimeofday (&tv, NULL); + struct tm* ptm = localtime(&tv.tv_sec); + /* Format the date and time, down to a single second. */ + char time_string[40]; + strftime (time_string, sizeof (time_string), "%Y-%m-%d %H:%M:%S", ptm); + /* Compute milliseconds from microseconds. */ + long milliseconds = tv.tv_usec / 1000; + /* Print the formatted time, in seconds, followed by a decimal point + and the milliseconds. */ + printf ("%s.%03ld\n", time_string, milliseconds); +} void Service::start(const std::string& args) { - printf("Hello world - OS included!\n"); + printf("Hello world! Time is now "); print_time(); printf("Args = %s\n", args.c_str()); printf("Try giving the service less memory, eg. 5MB in vm.json\n"); } diff --git a/src/musl/clock_gettime.cpp b/src/musl/clock_gettime.cpp index 29e0544bb7..a1ea981137 100644 --- a/src/musl/clock_gettime.cpp +++ b/src/musl/clock_gettime.cpp @@ -1,7 +1,23 @@ #include "common.hpp" +long sys_clock_gettime(clockid_t clk_id, struct timespec* tp) +{ + if (clk_id == CLOCK_REALTIME) + { + *tp = __arch_wall_clock(); + return 0; + } + else if (clk_id == CLOCK_MONOTONIC) + { + uint64_t ts = __arch_system_time(); + tp->tv_sec = ts / 1000000000ull; + tp->tv_nsec = ts % 1000000000ull; + return 0; + } + return -EPERM; +} + extern "C" -long syscall_SYS_clock_gettime() { - //STUB("clock_gettime"); - return 0; +long syscall_SYS_clock_gettime(clockid_t clk_id, struct timespec* tp) { + return strace(sys_clock_gettime, "clock_gettime", clk_id, tp); } diff --git a/src/musl/gettimeofday.cpp b/src/musl/gettimeofday.cpp index 5dfa11319a..4d853e2bf5 100644 --- a/src/musl/gettimeofday.cpp +++ b/src/musl/gettimeofday.cpp @@ -1,7 +1,19 @@ #include "common.hpp" +#include -extern "C" -long syscall_SYS_gettimeofday() { - STUB("gettimeofday"); +static long sys_gettimeofday(struct timeval *tv, struct timezone *tz) +{ + timespec t = __arch_wall_clock(); + tv->tv_sec = t.tv_sec; + tv->tv_usec = t.tv_nsec * 1000; + if (tz != nullptr) { + tz->tz_minuteswest = 0; + tz->tz_dsttime = 0; /* DST_NONE */ + } return 0; } + +extern "C" +long syscall_SYS_gettimeofday(struct timeval *tv, struct timezone *tz) { + return strace(sys_gettimeofday, "gettimeofday", tv, tz); +} From 50957fc3031ce0e8c47b1a02ab26db00b14f7951 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 13 Apr 2018 11:51:08 +0200 Subject: [PATCH 182/723] LiveUpdate: Use OS::heap_end() etc. --- lib/LiveUpdate/resume.cpp | 7 ++----- lib/LiveUpdate/update.cpp | 7 +------ 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/lib/LiveUpdate/resume.cpp b/lib/LiveUpdate/resume.cpp index 916fb9ae69..3862feb450 100644 --- a/lib/LiveUpdate/resume.cpp +++ b/lib/LiveUpdate/resume.cpp @@ -29,9 +29,6 @@ //#define LPRINT(x, ...) printf(x, ##__VA_ARGS__); #define LPRINT(x, ...) /** x **/ -// heap area -extern char* heap_end; - namespace liu { static bool resume_begin(storage_header&, std::string, LiveUpdate::resume_func); @@ -59,10 +56,10 @@ bool LiveUpdate::resume(std::string key, resume_func func) { void* location = OS::liveupdate_storage_area(); /// memory sanity check - if (heap_end >= (char*) location) { + if (OS::heap_end() >= (uintptr_t) location) { fprintf(stderr, "WARNING: LiveUpdate storage area inside heap (margin: %ld)\n", - (long int) (heap_end - (char*) location)); + (long int) (OS::heap_end() - (uintptr_t) location)); throw std::runtime_error("LiveUpdate storage area inside heap"); } return resume_helper(location, std::move(key), func); diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index 5051ead775..c2de1c7d00 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -25,9 +25,7 @@ #include #include #include -#include #include -#include #include "storage.hpp" #include #include @@ -50,9 +48,6 @@ extern "C" void* __os_store_soft_reset(const void*, size_t); // kernel area extern char _ELF_START_; extern char _end; -// heap area -extern char* heap_begin; -extern char* heap_end; // turn this off to reduce liveupdate times at the cost of extra checks bool LIVEUPDATE_PERFORM_SANITY_CHECKS = true; @@ -116,7 +111,7 @@ void LiveUpdate::exec(const buffer_t& blob) if (storage_area >= &_ELF_START_ && storage_area < &_end) { throw std::runtime_error("LiveUpdate storage area is inside kernel area"); } - if (storage_area >= heap_begin && storage_area < heap_end) { + if (storage_area >= (char*) OS::heap_begin() && storage_area < (char*) OS::heap_end()) { throw std::runtime_error("LiveUpdate storage area is inside the heap area"); } if (storage_area >= (char*) OS::heap_max()) { From 9696ef8982efddfdc2e323e8599931beebc17beb Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 13 Apr 2018 15:46:21 +0200 Subject: [PATCH 183/723] musl: Use C locale explicitly --- src/musl/clock_gettime.cpp | 4 ++-- src/musl/sigmask.cpp | 6 ++---- src/platform/x86_pc/kernel_start.cpp | 11 ++++++----- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/musl/clock_gettime.cpp b/src/musl/clock_gettime.cpp index a1ea981137..6dfe8bfc5b 100644 --- a/src/musl/clock_gettime.cpp +++ b/src/musl/clock_gettime.cpp @@ -1,6 +1,6 @@ #include "common.hpp" -long sys_clock_gettime(clockid_t clk_id, struct timespec* tp) +static long sys_clock_gettime(clockid_t clk_id, struct timespec* tp) { if (clk_id == CLOCK_REALTIME) { @@ -14,7 +14,7 @@ long sys_clock_gettime(clockid_t clk_id, struct timespec* tp) tp->tv_nsec = ts % 1000000000ull; return 0; } - return -EPERM; + return -EINVAL; } extern "C" diff --git a/src/musl/sigmask.cpp b/src/musl/sigmask.cpp index b5da2816de..3d9ff6455e 100644 --- a/src/musl/sigmask.cpp +++ b/src/musl/sigmask.cpp @@ -12,7 +12,7 @@ static int sys_sigmask(int /*signum*/) extern const bool __strace; -extern "C" { +extern "C" int syscall_SYS_rt_sigprocmask(int how, const sigset_t *set, sigset_t *oldset) { if constexpr (__strace) @@ -39,9 +39,7 @@ int syscall_SYS_rt_sigprocmask(int how, const sigset_t *set, sigset_t *oldset) { return sys_rt_sigprocmask(how, set, oldset); } - +extern "C" long syscall_SYS_sigmask(int signum) { return stubtrace(sys_sigmask, "sigmask", signum); } - -} diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 7c6025ce90..e3825027d7 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -204,7 +204,7 @@ void kernel_start(uint32_t magic, uint32_t addr) aux[i++].set_ptr(AT_PLATFORM, plat); aux[i++].set_long(AT_NULL, 0); - std::array argv; + std::array argv; // Parameters to main argv[0] = (char*) Service::name(); @@ -212,11 +212,12 @@ void kernel_start(uint32_t magic, uint32_t addr) int argc = 1; // Env vars - argv[2] = strdup("USER=root"); - argv[3] = 0x0; + argv[2] = strdup("LC_CTYPE=C"); + argv[3] = strdup("LC_ALL=C"); + argv[4] = strdup("USER=root"); + argv[5] = 0x0; - memcpy(&argv[4], aux, sizeof(auxv_t) * 38); - Expects(elf.is_ELF() && "Elf header present"); + memcpy(&argv[6], aux, sizeof(auxv_t) * 38); #if defined(__x86_64__) kprintf("* Initialize syscall MSR\n"); From 85108c40d79a0bed8a0e0c963c40030a593f59c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 13 Apr 2018 16:17:44 +0200 Subject: [PATCH 184/723] musl: Avoid strftime causing errno being set --- api/util/isotime.hpp | 4 ++++ src/util/syslogd.cpp | 3 +++ 2 files changed, 7 insertions(+) diff --git a/api/util/isotime.hpp b/api/util/isotime.hpp index a6d6fb9b16..cd88c6e251 100644 --- a/api/util/isotime.hpp +++ b/api/util/isotime.hpp @@ -21,6 +21,7 @@ #include #include +#include struct isotime { @@ -36,8 +37,11 @@ struct isotime */ static std::string to_datetime_string(time_t ts) { + // musl bandaid, bypass strftime setting errno + const int errno_save = errno; char buf[sizeof("2017-04-10T13:37:00Z")]; const auto res = std::strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&ts)); + errno = errno_save; return {buf, res}; } diff --git a/src/util/syslogd.cpp b/src/util/syslogd.cpp index 4c52e53590..91379a9830 100644 --- a/src/util/syslogd.cpp +++ b/src/util/syslogd.cpp @@ -32,6 +32,8 @@ void Syslog::syslog(const int priority, const char* fmt, ...) // va_list arguments (POSIX) void Syslog::syslog(const int priority, const char* fmt, va_list args) { + // due to musl "bug" (strftime setting errno..) + const int save_errno = errno; // snprintf removes % if calling syslog with %m in addition to arguments // Find %m here first and escape % if found std::regex m_regex{"\\%m"}; @@ -71,6 +73,7 @@ void Syslog::syslog(const int priority, const char* fmt, va_list args) specifier character is not just %m, the behavior is undefined. A trailing may be added if needed. */ + errno = save_errno; // Handle %m (replace it with strerror(errno)) and add the message (buf) message += std::regex_replace(buf, m_regex, strerror(errno)); From 73ebbf21b04711412f858bc97e520619a1d35b5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 13 Apr 2018 16:18:50 +0200 Subject: [PATCH 185/723] fd: Don't force socket handlers to be overriden --- api/posix/sockfd.hpp | 9 ++++++--- api/posix/udp_fd.hpp | 4 ---- src/posix/udp_fd.cpp | 14 -------------- 3 files changed, 6 insertions(+), 21 deletions(-) diff --git a/api/posix/sockfd.hpp b/api/posix/sockfd.hpp index 654bdda05e..eb55255e83 100644 --- a/api/posix/sockfd.hpp +++ b/api/posix/sockfd.hpp @@ -35,9 +35,12 @@ class SockFD : public FD { typedef delegate on_write_func; typedef delegate on_except_func; - virtual on_read_func get_default_read_func() = 0; - virtual on_write_func get_default_write_func() = 0; - virtual on_except_func get_default_except_func() = 0; + virtual on_read_func get_default_read_func() + { return [] (net::tcp::buffer_t) {}; } + virtual on_write_func get_default_write_func() + { return [] {}; } + virtual on_except_func get_default_except_func() + { return [] {}; } }; struct sockaddr; diff --git a/api/posix/udp_fd.hpp b/api/posix/udp_fd.hpp index a0c1c3e98e..c5ee5369d9 100644 --- a/api/posix/udp_fd.hpp +++ b/api/posix/udp_fd.hpp @@ -70,10 +70,6 @@ class UDP_FD : public SockFD { net::tcp::buffer_t buffer; }; - on_read_func get_default_read_func() override; - on_write_func get_default_write_func() override; - on_except_func get_default_except_func() override; - private: std::deque buffer_; // http://osr507doc.xinuos.com/en/netguide/disockD.connecting_datagrams.html diff --git a/src/posix/udp_fd.cpp b/src/posix/udp_fd.cpp index 10d17b7a1b..fc162e89d6 100644 --- a/src/posix/udp_fd.cpp +++ b/src/posix/udp_fd.cpp @@ -364,17 +364,3 @@ int UDP_FD::setsockopt(int level, int option_name, } // < switch(option_name) } -/// socket default handler getters - -UDP_FD::on_read_func UDP_FD::get_default_read_func() -{ - return [] (net::tcp::buffer_t) {}; -} -UDP_FD::on_write_func UDP_FD::get_default_write_func() -{ - return [] {}; -} -UDP_FD::on_except_func UDP_FD::get_default_except_func() -{ - return [] {}; -} From a2d95067a5741268a0a554960a5ab3c34a5c6462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 13 Apr 2018 16:20:59 +0200 Subject: [PATCH 186/723] musl: Start on a simple Unix socket abstraction --- api/posix/unix_fd.hpp | 44 ++++++++++++++++++++++++++ api/posix/unix_fd_impl.hpp | 30 ++++++++++++++++++ src/CMakeLists.txt | 3 +- src/musl/socketcall.cpp | 3 ++ src/posix/unix_fd.cpp | 63 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 1 deletion(-) create mode 100644 api/posix/unix_fd.hpp create mode 100644 api/posix/unix_fd_impl.hpp create mode 100644 src/posix/unix_fd.cpp diff --git a/api/posix/unix_fd.hpp b/api/posix/unix_fd.hpp new file mode 100644 index 0000000000..0a5ec35e7d --- /dev/null +++ b/api/posix/unix_fd.hpp @@ -0,0 +1,44 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef POSIX_UNIX_FD_HPP +#define POSIX_UNIX_FD_HPP + +#include "sockfd.hpp" +#include "unix_fd_impl.hpp" +#include + +class Unix_FD : public SockFD { +public: + using Impl = Unix_FD_impl; + Unix_FD(int id, int type) + : SockFD(id), type_(type) + {} + + /** SOCKET */ + long connect(const struct sockaddr *, socklen_t) override; + ssize_t sendto(const void* buf, size_t, int fl, + const struct sockaddr* addr, socklen_t addrlen) override; + int close() override; +private: + Impl* impl = nullptr; + const int type_; + + long set_impl_if_needed(const struct sockaddr* addr, socklen_t addrlen); +}; + +#endif diff --git a/api/posix/unix_fd_impl.hpp b/api/posix/unix_fd_impl.hpp new file mode 100644 index 0000000000..df20ba8ec7 --- /dev/null +++ b/api/posix/unix_fd_impl.hpp @@ -0,0 +1,30 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef POSIX_UNIX_FD_IMPL_HPP +#define POSIX_UNIX_FD_IMPL_HPP + +#include +class Unix_FD_impl { +public: + //virtual long connect(const struct sockaddr *, socklen_t) = 0; + virtual ssize_t sendto(const void* buf, size_t, int fl, + const struct sockaddr* addr, socklen_t addrlen) = 0; + virtual ~Unix_FD_impl() = default; +}; + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2fe8475316..9896130ae8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -60,7 +60,8 @@ set(OS_OBJECTS fs/disk.cpp fs/filesystem.cpp fs/dirent.cpp fs/mbr.cpp fs/path.cpp fs/fat.cpp fs/fat_async.cpp fs/fat_sync.cpp fs/memdisk.cpp # POSIX - posix/fd.cpp posix/file_fd.cpp posix/tcp_fd.cpp posix/udp_fd.cpp + posix/fd.cpp posix/file_fd.cpp posix/tcp_fd.cpp posix/udp_fd.cpp posix/unix_fd.cpp + ) add_library(os STATIC ${OS_OBJECTS}) diff --git a/src/musl/socketcall.cpp b/src/musl/socketcall.cpp index cae35a3297..49ca79668a 100644 --- a/src/musl/socketcall.cpp +++ b/src/musl/socketcall.cpp @@ -5,9 +5,12 @@ #include #include #include +#include static long sock_socket(int domain, int type, int protocol) { + if(domain == AF_UNIX) + return FD_map::_open(type).get_id(); // currently only support for AF_INET (IPv4, no local/unix or IP6) if (UNLIKELY(domain != AF_INET)) return -EAFNOSUPPORT; diff --git a/src/posix/unix_fd.cpp b/src/posix/unix_fd.cpp new file mode 100644 index 0000000000..448cbcee8e --- /dev/null +++ b/src/posix/unix_fd.cpp @@ -0,0 +1,63 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +long Unix_FD::set_impl_if_needed(const struct sockaddr* addr, socklen_t addrlen) +{ + if((addr == nullptr and addrlen == 0) and impl != nullptr) + return 0; + + if(UNLIKELY(addr == nullptr)) + return -EINVAL; + + const auto& un_addr = reinterpret_cast(*addr); + + if(un_addr.sun_family != AF_UNIX) + return -EAFNOSUPPORT; + + const std::string path{un_addr.sun_path}; + + try { + auto& ent = fs::VFS::get(path); + impl = &ent; + return 0; + } + catch (const fs::Err_not_found& e) { + printf("no ent: %s\n", e.what()); + return -ENOENT; + } +} + +long Unix_FD::connect(const struct sockaddr* addr, socklen_t addrlen) +{ + return set_impl_if_needed(addr, addrlen); +} + +ssize_t Unix_FD::sendto(const void* buf, size_t len, int fl, + const struct sockaddr* addr, + socklen_t addrlen) +{ + auto res = set_impl_if_needed(addr, addrlen); + return (res < 0) ? res : impl->sendto(buf, len, fl, addr, addrlen); +} + +int Unix_FD::close() +{ + return 0; +} From d26c8ee5f2e8ac0a07caf6bf9db9f5c5b05ef0aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 13 Apr 2018 16:23:24 +0200 Subject: [PATCH 187/723] musl: Created simple syslog backend for print to stdout, simplified test --- api/posix/syslog_print_socket.hpp | 40 +++++++++++++++ .../integration/syslog_default/service.cpp | 32 +++++++----- test/posix/integration/syslog_default/test.py | 50 +++++++------------ 3 files changed, 79 insertions(+), 43 deletions(-) create mode 100644 api/posix/syslog_print_socket.hpp diff --git a/api/posix/syslog_print_socket.hpp b/api/posix/syslog_print_socket.hpp new file mode 100644 index 0000000000..cb9c4f97e2 --- /dev/null +++ b/api/posix/syslog_print_socket.hpp @@ -0,0 +1,40 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef POSIX_SYSLOG_PRINT_SOCKET_HPP +#define POSIX_SYSLOG_PRINT_SOCKET_HPP + +#include + +class Syslog_print_socket : public Unix_FD_impl { +public: + inline ssize_t sendto(const void* buf, size_t, int fl, + const struct sockaddr* addr, socklen_t addrlen) override; + + ~Syslog_print_socket() = default; +}; + +#include +ssize_t Syslog_print_socket::sendto(const void* buf, size_t len, int /*fl*/, + const struct sockaddr* /*addr*/, + socklen_t /*addrlen*/) +{ + OS::print(reinterpret_cast(buf), len); + return len; +} + +#endif diff --git a/test/posix/integration/syslog_default/service.cpp b/test/posix/integration/syslog_default/service.cpp index 8d2ba088c4..613429f8ea 100644 --- a/test/posix/integration/syslog_default/service.cpp +++ b/test/posix/integration/syslog_default/service.cpp @@ -22,42 +22,47 @@ /* For testing IncludeOS */ #include - +#include +#include int main() { + static Syslog_print_socket syslog_sock_impl; + const fs::Path path{"/dev/log"}; + printf("path: %s\n", path.to_string().c_str()); + fs::VFS::mount(path, syslog_sock_impl, "Just a printing syslog"); /* ----------- Integration test for the default syslog behavior: printf ----------- */ /* ------------------------- Testing POSIX syslog ------------------------- */ - + INFO("Syslog", "POSIX"); int invalid_priority = -1; - syslog(invalid_priority, "Invalid %d", invalid_priority); - invalid_priority = 10; syslog(invalid_priority, "Invalid %d", invalid_priority); - invalid_priority = 55; - syslog(invalid_priority, "Invalid %d", invalid_priority); + syslog(LOG_INFO, " : A info message"); std::string one{"one"}; std::string two{"two"}; - size_t number = 33; - - syslog(LOG_INFO, "No open has been called prior to this"); syslog(LOG_NOTICE, "Program created with two arguments: %s and %s", one.c_str(), two.c_str()); + INFO2("Adding prepend"); + openlog("Prepended message", 0, LOG_MAIL); syslog(LOG_ERR, "Log after prepended message with one argument: %d", 44); - syslog(LOG_WARNING, "Log number two after openlog set prepended message"); closelog(); + size_t number = 33; syslog(LOG_WARNING, "Log after closelog with three arguments. One is %u, another is %s, a third is %d", number, "this", 4011); openlog("Second prepended message", LOG_PID | LOG_NDELAY, LOG_USER); syslog(LOG_EMERG, "Emergency log after openlog and new facility: user"); + errno = EINVAL; syslog(LOG_ALERT, "Alert log with the m argument: %m"); + errno = 0; + syslog(LOG_ALERT, "Second alert log with the m argument: %m"); + closelog(); @@ -65,6 +70,7 @@ int main() closelog(); + // max 32 chars ident allowed openlog("Open after close prepended message", LOG_PERROR, LOG_KERN); syslog(LOG_INFO, "Info after openlog with both m: %m and two hex arguments: 0x%x and 0x%x", 100, 50); @@ -72,7 +78,7 @@ int main() closelog(); /* ------------------------- Testing IncludeOS syslog ------------------------- */ - + INFO("Syslog", "Syslogd"); invalid_priority = -1; Syslog::syslog(invalid_priority, "Syslogd Invalid %d", invalid_priority); @@ -92,7 +98,7 @@ int main() Syslog::closelog(); - Syslog::syslog(LOG_WARNING, "Syslogd Log after closelog with three arguments. One is %u, another is %s, a third is %d", number, "this", 4011); + Syslog::syslog(LOG_WARNING, "Syslogd Log after closelog with three arguments. One is %zu, another is %s, a third is %d", number, "this", 4011); Syslog::openlog("Second prepended message", LOG_PID | LOG_NDELAY, LOG_USER); @@ -107,7 +113,9 @@ int main() Syslog::openlog("Open after close prepended message", LOG_PERROR, LOG_KERN); + errno = 0; Syslog::syslog(LOG_INFO, "Syslogd Info after openlog with both m: %m and two hex arguments: 0x%x and 0x%x", 100, 50); + printf("%d\n", errno); Syslog::openlog("Exiting test", 0, LOG_LOCAL7); diff --git a/test/posix/integration/syslog_default/test.py b/test/posix/integration/syslog_default/test.py index 25b57d5e13..4a158082c0 100755 --- a/test/posix/integration/syslog_default/test.py +++ b/test/posix/integration/syslog_default/test.py @@ -29,7 +29,10 @@ def increment(line): num_outputs += 1 print "num_outputs after increment: ", num_outputs -expected_outputs = 53 +def unexpected(line): + assert False + +expected_outputs = 22 def check_num_outputs(line): print "Registered", num_outputs, " / ", expected_outputs, " expected ouput lines" @@ -37,45 +40,30 @@ def check_num_outputs(line): vmrunner.vms[0].exit(0, "All tests passed") # ---------- POSIX wrapper syslog ---------- +vm.on_output(" : Invalid -1", unexpected) -vm.on_output("<11> " + ERR_C + " " + END_C, increment) -vm.on_output(" test_syslog_default: Syslog: Unknown priority -1. Message: Invalid -1", increment) - -# Count 1. vm.on_output("<11> " + ERR_C + " " + END_C, increment) -vm.on_output(" test_syslog_default: Syslog: Unknown priority 10. Message: Invalid 10", increment) - -# Count 1. vm.on_output("<11> " + ERR_C + " " + END_C, increment) -vm.on_output(" test_syslog_default: Syslog: Unknown priority 55. Message: Invalid 55", increment) +vm.on_output(" : A info message", increment) -vm.on_output("<14> " + INFO_C + " " + END_C, increment) -vm.on_output(" test_syslog_default: No open has been called prior to this", increment) +vm.on_output(" : Program created with two arguments: one and two", increment) -vm.on_output("<13> " + NOTICE_C + " " + END_C, increment) -vm.on_output(" test_syslog_default: Program created with two arguments: one and two", increment) +vm.on_output(" Prepended message: Log after prepended message with one argument: 44", increment) -vm.on_output("<19> " + ERR_C + " " + END_C, increment) -vm.on_output(" test_syslog_default Prepended message: Log after prepended message with one argument: 44", increment) +vm.on_output(" Prepended message: Log number two after openlog set prepended message", increment) -vm.on_output("<20> " + WARNING_C + " " + END_C, increment) -vm.on_output(" test_syslog_default Prepended message: Log number two after openlog set prepended message", increment) - -vm.on_output("<12> " + WARNING_C + " " + END_C, increment) -vm.on_output(" test_syslog_default: Log after closelog with three arguments. " + +vm.on_output(" Prepended message: Log after closelog with three arguments. " + "One is 33, another is this, a third is 4011", increment) -vm.on_output("<8> " + EMERG_C + " " + END_C, increment) -vm.on_output(" test_syslog_default Second prepended message\\[1\\]: Emergency log after openlog and new facility: user", increment) +vm.on_output(" Second prepended message\\[1\\]: Emergency log after openlog and new facility: user", increment) -vm.on_output("<9> " + ALERT_C + " " + END_C, increment) -vm.on_output(" test_syslog_default Second prepended message\\[1\\]: Alert log with the m argument: Success", increment) +vm.on_output(" Second prepended message\\[1\\]: Alert log with the m argument: Invalid argument", increment) -vm.on_output("<10> " + CRIT_C + " " + END_C, increment) -vm.on_output(" test_syslog_default: Critical after cleared prepended message", increment) +vm.on_output(" Second prepended message\\[1\\]: Second alert log with the m argument: No error information", increment) -# Count 2. Also has logopt LOG_PERROR (so will also be written to std::cerr) -vm.on_output("<6> " + INFO_C + " " + END_C, increment) -vm.on_output(" test_syslog_default Open after close prepended message: " + - "Info after openlog with both m: Success and two hex arguments: 0x64 and 0x32", increment) +vm.on_output(" Second prepended message\\[1\\]: Critical after cleared prepended message", increment) + +# std err is just a regular printf +vm.on_output("Open after close prepended mess: " + + "Info after openlog with both m: No error information and two hex arguments: 0x64 and 0x32", increment) # ---------- IncludeOS syslogd ---------- @@ -116,7 +104,7 @@ def check_num_outputs(line): # Count 2. Also has logopt LOG_PERROR (so will also be written to std::cerr) # Count 1. vm.on_output("<6> " + INFO_C + " " + END_C, increment) vm.on_output(" test_syslog_default Open after close prepended message: " + - "Syslogd Info after openlog with both m: Success and two hex arguments: 0x64 and 0x32", increment) + "Syslogd Info after openlog with both m: No error information and two hex arguments: 0x64 and 0x32", increment) vm.on_output("<191> " + DEBUG_C + " " + END_C, increment) vm.on_output(" test_syslog_default Exiting test: Something special to close with", check_num_outputs) From 2d646faf9141f60fe19eec2d54abd0bdc41deefa Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 13 Apr 2018 17:18:07 +0200 Subject: [PATCH 188/723] alloc_lstack: keep track of highest allocated address --- api/util/alloc_lstack.hpp | 38 +++++++++++++++++++++++- test/util/unit/lstack.cpp | 61 ++++++++++++++++++++++++++++++++------- 2 files changed, 87 insertions(+), 12 deletions(-) diff --git a/api/util/alloc_lstack.hpp b/api/util/alloc_lstack.hpp index 896e743b2f..d6765fe023 100644 --- a/api/util/alloc_lstack.hpp +++ b/api/util/alloc_lstack.hpp @@ -43,6 +43,7 @@ namespace alloc { * * NOTE: The allocator will not search for duplicates on deallocation, * e.g. won't prevent double free, so that has to be done elsewhere. + * It also won't join contigous blocks on deallocation. **/ template class Lstack { @@ -63,19 +64,32 @@ class Lstack { void* allocate(size_t size){ Expects((size & (Min - 1)) == 0); auto* chunk = find_free(size); - if (chunk != nullptr) + if (chunk != nullptr) { bytes_allocated_ += chunk->size; + + auto addr_end = (uintptr_t)chunk + chunk->size; + if (UNLIKELY(addr_end > allocation_end_)) { + allocation_end_ = addr_end; + } + } return chunk; } void deallocate (void* ptr, size_t size){ push(ptr, size); bytes_allocated_ -= size; + auto addr_end = (uintptr_t) ptr + size; + if (UNLIKELY(addr_end >= allocation_end_)) { + allocation_end_ = (uintptr_t) ptr; + } } void donate(void* ptr, size_t size){ push(ptr, size); bytes_total_ += size; + if ((uintptr_t)ptr > allocation_end_) { + allocation_end_ = (uintptr_t)ptr; + } } const Chunk* begin() const { @@ -90,6 +104,27 @@ class Lstack { size_t bytes_free() { return bytes_total_ - bytes_allocated_; } + /** + * The highest address allocated (more or less ever) + 1. + * No memory has been handed out beyond this point, but this is likely + * a very pessimistic estimate. To get highest actually allocated address, + * iterate over the chunks. + **/ + uintptr_t allocation_end() { + return allocation_end_; + } + + ssize_t chunk_count(){ + ssize_t res = 0; + auto* next = front_; + do { + if (next == nullptr) + return res; + res++; + } while ((next = next->next)); + return res; + } + private: Chunk* new_chunk(void* addr_begin, Chunk* next, size_t sz){ Expects((sz & align - 1) == 0); @@ -140,6 +175,7 @@ class Lstack { Chunk* front_ = nullptr; ssize_t bytes_allocated_ = 0; ssize_t bytes_total_ = 0; + uintptr_t allocation_end_ = 0; }; } // namespace util diff --git a/test/util/unit/lstack.cpp b/test/util/unit/lstack.cpp index 13c5dbeb08..06ca829786 100644 --- a/test/util/unit/lstack.cpp +++ b/test/util/unit/lstack.cpp @@ -56,6 +56,7 @@ CASE("Using lstack") { Lstack heap; + EXPECT(heap.allocation_end() == 0); EXPECT(heap.allocate(rand() & ~4095) == nullptr); auto poolsize = 0x100000; @@ -63,19 +64,25 @@ CASE("Using lstack") char* pool = (char*)memalign(blocksize, poolsize); void* pool_end = pool + poolsize; + + // Donate pool heap.donate(pool, poolsize); EXPECT(heap.bytes_allocated() == 0); EXPECT(heap.bytes_free() == poolsize); + EXPECT(heap.allocation_end() == (uintptr_t)pool); + // First allocation auto* page = heap.allocate(blocksize); EXPECT(page == pool); EXPECT(((Chunk*)page)->size == blocksize); EXPECT(((Chunk*)page)->next == nullptr); EXPECT(heap.bytes_allocated() == blocksize); EXPECT(heap.bytes_free() == poolsize - blocksize); - + EXPECT(heap.allocation_end() == (uintptr_t)pool + blocksize); + EXPECT(heap.chunk_count() == 1); print_summary(heap.begin()); + // Empty out the pool int i = 0; auto prev_bytes_alloc = heap.bytes_allocated(); for (; i < poolsize - blocksize; i += blocksize) @@ -85,17 +92,21 @@ CASE("Using lstack") EXPECT(heap.bytes_allocated() == prev_bytes_alloc + blocksize); prev_bytes_alloc = heap.bytes_allocated(); EXPECT(heap.bytes_free() == poolsize - prev_bytes_alloc); + EXPECT(heap.allocation_end() == (uintptr_t)p + blocksize); } print_summary(heap.begin()); - EXPECT(heap.allocate(blocksize) == nullptr); EXPECT(heap.begin() == nullptr); + EXPECT(heap.chunk_count() == 0); + auto heap_end = heap.allocation_end(); + // First deallocation char* chunk1 = pool + 0x1000; heap.deallocate(chunk1, 0x2000); EXPECT((char*)heap.begin() == chunk1); EXPECT((char*)heap.begin()->next == nullptr); + EXPECT(heap.allocation_end() == heap_end); print_summary(heap.begin()); @@ -104,6 +115,7 @@ CASE("Using lstack") EXPECT((char*)heap.begin() == chunk2); EXPECT((char*)heap.begin()->next == chunk1); EXPECT((char*)heap.begin()->next->next == nullptr); + EXPECT(heap.allocation_end() == heap_end); print_summary(heap.begin()); EXPECT(heap.allocate(0x4000) == chunk2); @@ -111,15 +123,18 @@ CASE("Using lstack") EXPECT(heap.allocate(0x1000) == chunk1); EXPECT(heap.allocate(0x1000) == chunk1 + 0x1000); EXPECT(heap.begin() == nullptr); + EXPECT(heap.allocation_end() == heap_end); // Free some small chunks in random order - void* prev = nullptr; std::vector rands; - auto size = blocksize * 2; - auto count = poolsize / size; + auto size2 = blocksize * 2; + auto count = poolsize / size2; auto index = 0; - for (i = 0; i < 5; i++) + std::array deallocs; + + // Create random chunks + for (i = 0; i < deallocs.size(); i++) { do { @@ -128,12 +143,23 @@ CASE("Using lstack") rands.push_back(index); - auto* frag = &(pool)[index * size]; - EXPECT((frag >= pool && frag <= pool + poolsize - size)); + auto* frag = &(pool)[index * size2]; + EXPECT((frag >= pool && frag <= pool + poolsize - size2)); EXPECT(((uintptr_t)frag & blocksize - 1 ) == 0); - heap.donate(frag, size); + deallocs[i] = frag; + } + + // Deallocate + for (auto& frag : deallocs) { + heap.deallocate(frag, size2); + + if ((uintptr_t)frag + size2 >= heap_end) { + EXPECT(heap.allocation_end() == (uintptr_t)frag); + } else { + EXPECT(heap.allocation_end() == heap_end); + } + EXPECT((void*)heap.begin() == frag); - prev = frag; print_summary(heap.begin()); } @@ -156,19 +182,32 @@ CASE("Using lstack") // Fragment the first chunk chunk1 = (char*)heap.allocate(0x1000); - EXPECT(chunk1 == pool + (rands.back() * size)); + EXPECT(chunk1 == pool + (rands.back() * size2)); print_summary(heap.begin()); heap.deallocate(chunk1, 0x1000); print_summary(heap.begin()); EXPECT(heap.begin() == (Chunk*)chunk1); EXPECT(heap.begin()->next->size == 0x1000); EXPECT(heap.begin()->next->next->size == 0x2000); + EXPECT(heap.allocation_end() == heap_end); auto* first_2k = heap.begin()->next->next; chunk2 = (char*)heap.allocate(0x2000); print_summary(heap.begin()); EXPECT((Chunk*)chunk2 == first_2k); + // hand back the last two chunks, expect allocation_end decrease + auto* minus_1 = pool + poolsize - blocksize; + auto* minus_2 = minus_1 - blocksize; + + EXPECT(heap_end > (uintptr_t)minus_1); + EXPECT(heap_end > (uintptr_t)minus_2); + + heap.deallocate(minus_1, blocksize); + EXPECT(heap.allocation_end() == (uintptr_t)minus_1); + heap.deallocate(minus_2, blocksize); + EXPECT(heap.allocation_end() == (uintptr_t)minus_2); + free(pool); From 8fad6d4ab53d11ac1a2da6deae52db19c22ddc60 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 13 Apr 2018 17:20:38 +0200 Subject: [PATCH 189/723] heap: heap_end() now asks mmap's allocator --- src/kernel/heap.cpp | 6 +++--- src/musl/mmap.cpp | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/kernel/heap.cpp b/src/kernel/heap.cpp index 47e22c3987..0e2c94cf9b 100644 --- a/src/kernel/heap.cpp +++ b/src/kernel/heap.cpp @@ -20,9 +20,9 @@ size_t brk_bytes_used(); size_t mmap_bytes_used(); +size_t mmap_allocation_end(); uintptr_t OS::heap_begin_ = 0; -uintptr_t OS::heap_end_ = __arch_max_canonical_addr; uintptr_t OS::heap_max_ = __arch_max_canonical_addr; uintptr_t OS::memory_end_ = __arch_max_canonical_addr;; @@ -53,7 +53,7 @@ uintptr_t OS::heap_begin() noexcept } uintptr_t OS::heap_end() noexcept { - return heap_end_; + return mmap_allocation_end(); } uintptr_t OS::resize_heap(size_t size) @@ -83,5 +83,5 @@ void OS::init_heap(uintptr_t free_mem_begin, uintptr_t memory_end) noexcept { heap_max_ = memory_end; heap_begin_ = util::bits::roundto(free_mem_begin); auto brk_end = __init_brk(heap_begin_); - heap_end_ = __init_mmap(brk_end); + __init_mmap(brk_end); } diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 84e4fbd907..694d194333 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -38,6 +38,10 @@ size_t mmap_bytes_free() { return alloc.bytes_free(); } +uintptr_t mmap_allocation_end(){ + return alloc.allocation_end(); +} + static void* sys_mmap(void *addr, size_t length, int /*prot*/, int /*flags*/, int fd, off_t /*offset*/) { From fc2f8e7ef09a8835e836fd7234c348cb9fadd233 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 13 Apr 2018 17:21:31 +0200 Subject: [PATCH 190/723] test: prevent optimizing out mallocs in stress test --- test/stress/service.cpp | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/test/stress/service.cpp b/test/stress/service.cpp index 026f1d45a7..9fa008cf1e 100644 --- a/test/stress/service.cpp +++ b/test/stress/service.cpp @@ -69,15 +69,20 @@ const std::string NOT_FOUND = "HTTP/1.1 404 Not Found \n Connection: close\n\n"; uint64_t TCP_BYTES_RECV = 0; uint64_t TCP_BYTES_SENT = 0; +void print_memuse(uintptr_t u) { + auto end = OS::heap_end(); + printf("Current memory usage: %s (%zi b) heap_end: 0x%zx lstack chunks: (%s)\n", + util::Byte_r(u).to_string().c_str(), u, end, util::Byte_r(end).to_string().c_str()); +} + void Service::start(const std::string&) { using namespace util::literals; // Allocation / free spam to warm up auto initial_memuse = OS::heap_usage(); - printf("Current memory usage: %zi b, (%s) \n", initial_memuse, util::Byte_r(initial_memuse).to_string().c_str()); + print_memuse(initial_memuse); - - std::array allocs {}; + std::array allocs {}; auto chunksize = 1024 * 1024 * 5; printf("Exercising heap: incrementally allocating %zi x %i bytes \n", allocs.size(), chunksize); @@ -85,8 +90,13 @@ void Service::start(const std::string&) for (auto& ptr : allocs) { ptr = malloc(chunksize); Expects(ptr != nullptr); + memset((void*)ptr, '!', chunksize); + for (char* c = (char*)ptr; c < (char*)ptr + chunksize; c++) + Expects(*c == '!'); + printf("Allocated area: %p heap_end: %p\n", (void*)ptr, (void*)OS::heap_end()); auto memuse = OS::heap_usage(); - printf("Current memory usage: %zi b, (%s) \n", memuse, util::Byte_r(memuse).to_string().c_str()); + + print_memuse(memuse); Expects(memuse > initial_memuse); } @@ -97,9 +107,9 @@ void Service::start(const std::string&) printf("Deallocating \n"); for (auto& ptr : allocs) { - free(ptr); + free((void*)ptr); auto memuse = OS::heap_usage(); - printf("Current memory usage: %zi b, (%s) \n", memuse, util::Byte_r(memuse).to_string().c_str()); + print_memuse(memuse); Expects(memuse < high_memuse); Expects(memuse < prev_memuse); prev_memuse = memuse; From e7da25b93066672498d6e519c0d86f3d853c9ae1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Sat, 14 Apr 2018 14:27:38 +0200 Subject: [PATCH 191/723] linux: Implement latest musl changes --- linux/src/arch.cpp | 9 +++++++++ linux/src/os.cpp | 7 +++++++ src/kernel/os.cpp | 19 +++++++++---------- src/platform/x86_solo5/os.cpp | 10 ++++++---- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/linux/src/arch.cpp b/linux/src/arch.cpp index be819b3bb1..ff66afdce1 100644 --- a/linux/src/arch.cpp +++ b/linux/src/arch.cpp @@ -2,6 +2,15 @@ #include #include #include +# define weak_alias(name, aliasname) \ + extern __typeof (name) aliasname __attribute__ ((weak, alias (#name))); + +typedef void (*ctor_t) (); +extern "C" __attribute__(( visibility("hidden") )) void default_ctor() {} +ctor_t __plugin_ctors_start = default_ctor; +weak_alias(__plugin_ctors_start, __plugin_ctors_end); +ctor_t __service_ctors_start = default_ctor; +weak_alias(__service_ctors_start, __service_ctors_end); void* _ELF_START_; void* _ELF_END_; diff --git a/linux/src/os.cpp b/linux/src/os.cpp index c76b0b1164..0f798a340c 100644 --- a/linux/src/os.cpp +++ b/linux/src/os.cpp @@ -6,6 +6,7 @@ #include #include // mallinfo() #include +bool __libc_initialized = true; void OS::event_loop() { @@ -100,6 +101,12 @@ void OS::start(char* cmdline, uintptr_t) OS::cmdline = cmdline; } +// stdout +void OS::default_stdout(const char* text, size_t len) +{ + write(STDOUT_FILENO, text, len); +} + // system_log has no place on Linux because stdout goes --> pipe void SystemLog::initialize() {} diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 4b1907529f..f06a716ba2 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -86,17 +86,16 @@ const char* OS::cmdline_args() noexcept { return cmdline; } -extern char __service_ctors_start; -extern char __service_ctors_end; -extern char __plugin_ctors_start; -extern char __plugin_ctors_end; +typedef void (*ctor_t) (); +extern ctor_t __service_ctors_start; +extern ctor_t __service_ctors_end; +extern ctor_t __plugin_ctors_start; +extern ctor_t __plugin_ctors_end; -int __run_ctors(uintptr_t begin, uintptr_t end) +int __run_ctors(ctor_t* begin, ctor_t* end) { int i = 0; - for (; begin < end; begin += sizeof(void(*)()), i++) { - (*(void (**)(void)) begin )(); - } + for (; begin < end; begin++, i++) (*begin)(); return i; } @@ -141,7 +140,7 @@ void OS::post_start() MYINFO("Initializing plugins"); // Run plugin constructors - __run_ctors((uintptr_t)&__plugin_ctors_start, (uintptr_t)&__plugin_ctors_end); + __run_ctors(&__plugin_ctors_start, &__plugin_ctors_end); // Run plugins @@ -157,7 +156,7 @@ void OS::post_start() OS::boot_sequence_passed_ = true; // Run service constructors - __run_ctors((uintptr_t)&__service_ctors_start, (uintptr_t)&__service_ctors_end); + __run_ctors(&__service_ctors_start, &__service_ctors_end); PROFILE("Service::start"); // begin service start diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index 9eee9051ab..d0d125cc2e 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -14,9 +14,6 @@ extern "C" { static uint64_t os_cycles_hlt = 0; extern "C" void* get_cpu_esp(); -extern char __init_array_start; -extern char __init_array_end; -int __run_ctors(uintptr_t begin, uintptr_t end); extern uintptr_t _start; extern uintptr_t _end; extern uintptr_t mem_size; @@ -27,6 +24,11 @@ extern uintptr_t _ELF_END_; // in kernel/os.cpp extern bool os_default_stdout; +typedef void (*ctor_t) (); +extern ctor_t __init_array_start; +extern ctor_t __init_array_end; +extern int __run_ctors(ctor_t* begin, ctor_t* end); + #define MYINFO(X,...) INFO("Kernel", X, ##__VA_ARGS__) #ifdef ENABLE_PROFILERS @@ -91,7 +93,7 @@ void OS::start(char* _cmdline, uintptr_t mem_size) // Call global ctors PROFILE("Global kernel constructors"); - __run_ctors((uintptr_t)&__init_array_start, (uintptr_t)&__init_array_end); + __run_ctors(&__init_array_start, &__init_array_end); PROFILE("Memory map"); // Assign memory ranges used by the kernel From a36620a10c4213c36afd87cca75878caa114a933 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 16 Apr 2018 12:39:28 +0200 Subject: [PATCH 192/723] openssl: Verify google.com in http example --- examples/http_client/CMakeLists.txt | 3 +- examples/http_client/certs/server.pem | 83 --------------------------- examples/http_client/config.json | 5 ++ examples/http_client/service.cpp | 2 + examples/http_client/vm.json | 2 +- src/net/openssl/client.cpp | 6 +- 6 files changed, 11 insertions(+), 90 deletions(-) delete mode 100644 examples/http_client/certs/server.pem create mode 100644 examples/http_client/config.json diff --git a/examples/http_client/CMakeLists.txt b/examples/http_client/CMakeLists.txt index 6fbfcc8118..8ec97480a6 100644 --- a/examples/http_client/CMakeLists.txt +++ b/examples/http_client/CMakeLists.txt @@ -16,13 +16,14 @@ set(BINARY "http_client") # Source files to be linked with OS library parts to form bootable image set(SOURCES service.cpp + ../acorn/fs/acorn_fs.cpp ) set(DRIVERS virtionet vmxnet3 e1000 - boot_logger + #boot_logger ) set(PLUGINS vfs) diff --git a/examples/http_client/certs/server.pem b/examples/http_client/certs/server.pem deleted file mode 100644 index 46190d9a4b..0000000000 --- a/examples/http_client/certs/server.pem +++ /dev/null @@ -1,83 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDMVU8mqaDVEPFG -zIKj9kNXTQB8RDjhVFg6xHm+3gXcp0PXyOqbWjsk9ZExIwAARo29nqWokNmisV3u -euBoUh2Ne/giP9Ir+76PViR3C+xvBUkI1YdTu+HkmLQDJLPKlzgF99PKUUh9467a -H0jWix59NyPSy1J9OVv+Z4V3bdCuiaPJI2qv+ysANYOmPQYQPWAINEo3xmNjRkx5 -j1QWgdhV2cmw4YjRCWgFzhzpSQYq0mP3AgZJ0L+hPoc2KSth/wSkLRS3q+lJTFTA -RUtw7w7icFQwRMiMNSGKvJJ9FMgNUsiAA6Ju6iHnBj9qd2mpDlJ+pV+3j0fFw2nK -7bL7Pjs9N9ninZJbigmpTieKPOkVuTV3DNCmWAqu5QtHGjYa8ySvzjBPJT83c/WU -G4eB8eUMjeq36d6YXurCLa/Ext1KUQidgDFNtys7YqqvtRHL45qC18hzlgzWy4o5 -UIOolS3pAkRorop65UXIV/lyn+i/SrzTcidQEB4DYI9DBWI7hMFSp6YHEaLrYE8I -9eQGlIhjsFJQTQTKwdzA83812BHkfeH2j6wuWAOnzTfztoKVisqbHFoybTtPw1It -ELxlMUuVQGJnoftwBipcfljlO3ukLXszQspppCSWqFeXF58IErhvkM9nOTLS9FWU -uGihqHEUZuUdjQfkYMZQ33dvTwL5DQIDAQABAoICAFMRf2sVXNgh2iTRMW8cx4Wv -PTpHteQNGQ1TILdypWlt6ovIF3Buv0qoNo2GUBLMnATxgSFFOeUFPE2eUKq4b2E8 -e8TYs0XEQ1UcTRjElSsRN/7KaAingg8f8VaxzhHXZiQu+z/tyY28STJNs5vc1NlM -2Fhy5icYcYi8ZysFTugV9qrnphDDCFk+aDXl6xghC3BqaZ3BHliWlLJ6jVHVGbDV -KESre6aXr11kzKYeVM6F17Psc8gAgFc6B1gCZquhNC7WUOzWi7GLwYyk3yWbWi6+ -QcuZuvodA3TWcqs9DpM0C2TXlWqH7p5wEMnkn+TleK4Z5w7FxgTlC5Y/ChtP0l63 -ElVjkOggliKCv6xmCAkVoG/7KaiFaTZgPZ+Mpl0S2toTdgWDQ/YnKxoJqX9AK03k -Tt5NGm6vdjQbb7V84uV1N1vl/DQXpI7E6teQagRIgXlnhPDlU5pl90Nx7HziUbdT -bHQ2yfL18M8/Q42u5o/+5jfB78VGe1tW3pznoybbkGdOWuhCldSypOL9pXShmLK+ -wKKQZomvw2J7SGW4yFnYcQ1avaaImtx/YQuLp/VazjXXU5miJ6hyxrxKLoNmZry8 -v/a/gzxqaWe9LgomCHBimBpOG6uf5rCV+5lMEjKfP8ka6Su+ZR8TITDIcjCnpNQo -MukSIiE2UaMA3Y9RSYbhAoIBAQD7XhwQB02JGOUjpYkwIIZDJYadRxBRm1cCYbPS -KHqLjHyVLgf1Iix8TzbOXvuy6rp2nmaaNP2L4uxNphpe3rK5LquG1rD/QHNt2vQx -PmDMiUcExNfEJa+x77G6S3PDyN3/kThJAHUBrnQmViur/kHFBDQT5H48HfroHL7O -V3+0U0O4FwPg7xo6m3lkTSMgxvW57LBzcSfII5qLh1+weVHWN5EgmwK6wlwtSZte -uBEdckZG0D2gfzxEn9VxHjiEll0YQjatwZ6KNamnQmVbXS3AEh41B2YVa6uAWAvT -Jm2gmlrN/6G3tqaFZBa0ZNIh5WmvHRGtJsimNrk/1urw+qhpAoIBAQDQGU2QM6/z -XuhrjkhKV1KiAbQAfEzsPCua3hQNZN10hMqhgC3KM2rF08NE9QOQShmFDIyxMQ7n -7iMED9g7sgwdXRTPrkhRj5Kumw3Mxx9P7tOP7I3HgHCyJGrsOePArgEELomF47Ye -vlGIGOVJkGRkQKxH1GMFKTApQRobCsla9pFkoHwFuPvQbgIY5HSZHEFzkr42Z1me -/7P9sHlVpiJS2iSY9c/v1ijKoElANPhWSewHXU5JS/ykQ6RM4+1Ezz9eOYFB66nI -/64AYLtZ/12KfaU8v2nL3jRgWbekj60jwYwtvgStKq3bmoa77pWd13OEQoYQ1TWq -3yNf77CHklcFAoIBAQDlUzBq9x6nqwyxfr4wKBwVelDbgA/FQ+EXjSdO3hse4SZv -uzULUQggyOwJTuJ2kmtW+KBGo32Fd5t6I8X/M1D+XTOjqo6D8LreSFlr66rIL52Z -HjRzAKbPwQv2GAwfXEwcctJU15Pn2s6ggxEIssgyA+m3FEiOdBOKW19ARNd/Hk+z -zVt+hOT6/wi6lx0rNBjObL92Mo/GqaS/Etwb1jLxKnS+uGy/Eb6imnqi6W7D3JN8 -21K98BZ9zQwfTSDm5X89OHtF/lSzZs3GiG6L8fBhQKZKXtulQzuNtcXj9zUgxnF9 -anCReJNnMq78GNcUkYXfere7rpxz94z+Twa8nxGxAoIBAQCFKKvJKkncoU54BIOd -jmMB57UW2nMedyIv5wtNxs0uUgqXbT3ftScNj4PUDDRLkB7kVDNy7sFrB6bAYqMr -D7NQ1s8KVatRBV4G/JDICGKw81HNNsFCd52lYEhLAIAMO0vLMTwuwJc49O+W9tdB -S3Px8rBLBkyWcNSwkUfzJePSoRRIH6k8wDHiHYKaCxXRI5N+MNgmV9uxMk4qd1Is -zDIKciwY9LIp9hAEGKzGuR5vmL2/uKTvof/33JC14uNgtH0H/iQH2pS6WWEuPLel -zxI8HEovpQw9t+IsBTzNwGI6Dw9z9bNoW3cEnsy4VOLILaVtbGP9QXEl9cLdELMV -DT9BAoIBAQCi1cFk2C7EluvHClbzlbUQBsHpCchisLgLF/appYCMMlZ55Ys2iQ2b -Q8xm4iNbB7r+kyBv2R7xnCPIM5dsEhWHmv1mfQY+sGZPBneTaSPV/4ZLG7YueepR -A52j5A6uXOt4R8T0te8GhhE7C0VcJU/i0vZQoAAQ5WGEi4SnO9dFhRAlcp+siIC7 -giiMuBQ82p2DWl48hpWJI+pLT+TmseFFZQ26GdFuDzJQxGF4fmRM30GHGXiiGU7S -8a+vXhA/w8SAaZhr0WadaD4bl5LvJUBuM+XVWr+ftS1VwVTEfuIAe4dFgNaDbc4Z -ucqyshD/YlGrku77JOcDw8KEexz1MeXE ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIFXTCCA0WgAwIBAgIJAL40szehpk5dMA0GCSqGSIb3DQEBDQUAMEUxCzAJBgNV -BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX -aWRnaXRzIFB0eSBMdGQwHhcNMTgwMjA1MjM0ODM4WhcNMjgwMjAzMjM0ODM4WjBF -MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 -ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC -CgKCAgEAzFVPJqmg1RDxRsyCo/ZDV00AfEQ44VRYOsR5vt4F3KdD18jqm1o7JPWR -MSMAAEaNvZ6lqJDZorFd7nrgaFIdjXv4Ij/SK/u+j1YkdwvsbwVJCNWHU7vh5Ji0 -AySzypc4BffTylFIfeOu2h9I1osefTcj0stSfTlb/meFd23QromjySNqr/srADWD -pj0GED1gCDRKN8ZjY0ZMeY9UFoHYVdnJsOGI0QloBc4c6UkGKtJj9wIGSdC/oT6H -NikrYf8EpC0Ut6vpSUxUwEVLcO8O4nBUMETIjDUhirySfRTIDVLIgAOibuoh5wY/ -andpqQ5SfqVft49HxcNpyu2y+z47PTfZ4p2SW4oJqU4nijzpFbk1dwzQplgKruUL -Rxo2GvMkr84wTyU/N3P1lBuHgfHlDI3qt+nemF7qwi2vxMbdSlEInYAxTbcrO2Kq -r7URy+OagtfIc5YM1suKOVCDqJUt6QJEaK6KeuVFyFf5cp/ov0q803InUBAeA2CP -QwViO4TBUqemBxGi62BPCPXkBpSIY7BSUE0EysHcwPN/NdgR5H3h9o+sLlgDp803 -87aClYrKmxxaMm07T8NSLRC8ZTFLlUBiZ6H7cAYqXH5Y5Tt7pC17M0LKaaQklqhX -lxefCBK4b5DPZzky0vRVlLhooahxFGblHY0H5GDGUN93b08C+Q0CAwEAAaNQME4w -HQYDVR0OBBYEFCS1cTijfysZ+WN0N67mR62VPezfMB8GA1UdIwQYMBaAFCS1cTij -fysZ+WN0N67mR62VPezfMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQENBQADggIB -AK3P0IurEm/AxIFrcbr4z1VVemZm/aMp2kU/x6UOKxnBh/IqrcCdYb5Ompr8NIML -dsra8+kaQD++sUNSyQfcYUTBmjatR756NfDSSSavywbutjTFM/0a1mID9xo4OaeQ -FoDI8OzU7rtxPWk+bn/C+vt1a0RzgE2xFFS1wxZoQ8UrLLdSoWAQMOVsk6H94H+j -kTZDRhYdC9Ij4Lo1zAPNH6BJT0AXFUZluteGsJlRG1y415mCTtPPogvXH88bLM2C -p1o0kMhrfXbJEQ3YHUbDjQ+E6vrTEe81Ym7RF2puSFakQAMF6USsHvQJY/AH2Ss1 -5amuGzu4Z1yLXY5Ln8uha8uTPHiMe0l82GNeBQZN3tv0b8npCP+8RJ1oBOoKurJJ -zVGemX1e61GHnK6L6mfZ78kYrbmc6O3IAGd4zEXRXKtIVyOhbVRrXzCA+NChMUdB -jX1QOCk5n9pJFELsgEseJLqqraRhtoqeQnMZ+F9c/J07LAovVdq7JHhM5DZj5RDr -KzIcu1456OLZWkIqt3gK2UoGL4KjzIju81n4PskjQ4L0QJQlTt1qVGU/lyBT+0vU -dRHGIRJ/IS1urrN/+w6XDiYN5Wynoo+FQFLcTRKsJuG8xzCJ1YcOn0fkkl7sqHTU -IITTjSEm0Cj8updtdoNrgNbedscZA5MxmgmPQZTzfcfM ------END CERTIFICATE----- diff --git a/examples/http_client/config.json b/examples/http_client/config.json new file mode 100644 index 0000000000..c42702a586 --- /dev/null +++ b/examples/http_client/config.json @@ -0,0 +1,5 @@ +{ + "vfs": [ + {"disk": "memdisk", "root": "/mozilla", "mount": "/certs", "description": ""} + ] +} diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index 9b4c92c435..1f0a7aaf9b 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -17,6 +17,7 @@ #include #include +#include "../acorn/fs/acorn_fs.hpp" static SSL_CTX* init_ssl_context() { @@ -25,6 +26,7 @@ static SSL_CTX* init_ssl_context() assert(!err); }); + acorn::list_static_content(disk.fs()); auto ents = disk.fs().ls("/mozilla"); // initialize client context diff --git a/examples/http_client/vm.json b/examples/http_client/vm.json index 17b60814b0..f45d601d13 100644 --- a/examples/http_client/vm.json +++ b/examples/http_client/vm.json @@ -1,5 +1,5 @@ { "net" : [ - {"device" : "virtio"} + {"device" : "virtio", "backend": "user"} ] } diff --git a/src/net/openssl/client.cpp b/src/net/openssl/client.cpp index d90d9e81ac..dafbf584bc 100644 --- a/src/net/openssl/client.cpp +++ b/src/net/openssl/client.cpp @@ -58,7 +58,6 @@ tls_init_client(fs::List ents) int res = SSL_CTX_set_cipher_list(ctx, "AES256-SHA"); assert(res == 1); - /* X509_STORE* store = X509_STORE_new(); assert(store != nullptr); @@ -77,10 +76,8 @@ tls_init_client(fs::List ents) // assign CA store to CTX SSL_CTX_set_cert_store(ctx, store); - */ - (void) ents; - SSL_CTX_load_verify_locations(ctx, nullptr, "/certs"); + //SSL_CTX_load_verify_locations(ctx, nullptr, "/mozilla"); // create private key for client tls_private_key_for_ctx(ctx); @@ -96,7 +93,6 @@ extern "C" typedef int verify_cb_t(int, X509_STORE_CTX*); static verify_cb_t verify_cb; int verify_cb(int preverify, X509_STORE_CTX* ctx) { - printf("verify_cb\n"); X509* current_cert = X509_STORE_CTX_get_current_cert(ctx); if (preverify == 0) From 5e19105ba7aabf1ae93eaf979b45b7cec0dcd58f Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Mon, 16 Apr 2018 16:09:33 +0200 Subject: [PATCH 193/723] heap: limit default physical memory to avoid mmap collisions in legacy boot --- src/kernel/heap.cpp | 8 ++++++-- src/platform/x86_pc/kernel_start.cpp | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/kernel/heap.cpp b/src/kernel/heap.cpp index 0e2c94cf9b..5db56065ec 100644 --- a/src/kernel/heap.cpp +++ b/src/kernel/heap.cpp @@ -17,14 +17,18 @@ #include #include +#include +using namespace util::literals; size_t brk_bytes_used(); size_t mmap_bytes_used(); size_t mmap_allocation_end(); +static constexpr size_t default_max_mem = 2_GiB; + uintptr_t OS::heap_begin_ = 0; -uintptr_t OS::heap_max_ = __arch_max_canonical_addr; -uintptr_t OS::memory_end_ = __arch_max_canonical_addr;; +uintptr_t OS::heap_max_ = default_max_mem; +uintptr_t OS::memory_end_ = __arch_max_canonical_addr; size_t OS::heap_usage() noexcept diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 7c6025ce90..5277d7d612 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -126,7 +126,7 @@ void kernel_start(uint32_t magic, uint32_t addr) // Determine where free memory starts extern char _end; uintptr_t free_mem_begin = reinterpret_cast(&_end); - uintptr_t memory_end = __arch_max_canonical_addr; + uintptr_t memory_end = OS::memory_end(); if (magic == MULTIBOOT_BOOTLOADER_MAGIC) { free_mem_begin = _multiboot_free_begin(addr); From f8adc552ac004da4d2d0668eafa3861f85259040 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 16 Apr 2018 16:28:42 +0200 Subject: [PATCH 194/723] microLB: Use streams instead of direct TCP connections --- lib/microLB/micro_lb/balancer.cpp | 52 ++++++++++++++++-------------- lib/microLB/micro_lb/balancer.hpp | 26 +++++++-------- lib/microLB/micro_lb/serialize.cpp | 18 ++++++----- 3 files changed, 51 insertions(+), 45 deletions(-) diff --git a/lib/microLB/micro_lb/balancer.cpp b/lib/microLB/micro_lb/balancer.cpp index e0b61e4cd3..08b889b57d 100644 --- a/lib/microLB/micro_lb/balancer.cpp +++ b/lib/microLB/micro_lb/balancer.cpp @@ -13,7 +13,7 @@ #define INITIAL_SESSION_TIMEOUT 5s #define ROLLING_SESSION_TIMEOUT 60s -#define LB_VERBOSE 0 +#define LB_VERBOSE 1 #if LB_VERBOSE #define LBOUT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else @@ -31,7 +31,9 @@ namespace microLB { netin.tcp().listen(in_port, [this] (auto conn) { - if (conn != nullptr) this->incoming(conn); + if (conn != nullptr) { + this->incoming(std::make_unique (conn)); + } }); this->init_liveupdate(); @@ -54,9 +56,9 @@ namespace microLB { return this->signal; } - void Balancer::incoming(tcp_ptr conn) + void Balancer::incoming(net::Stream_ptr conn) { - queue.emplace_back(conn); + queue.emplace_back(std::move(conn)); LBOUT("Queueing connection (q=%lu)\n", queue.size()); // IMPORTANT: try to handle queue, in case its ready // don't directly call handle_connections() from here! @@ -70,7 +72,7 @@ namespace microLB auto& client = queue.front(); if (client.conn->is_connected()) { // NOTE: explicitly want to copy buffers - if (nodes.assign(client.conn, client.readq)) { + if (nodes.assign(std::move(client.conn), client.readq)) { queue.pop_front(); } } @@ -112,8 +114,8 @@ namespace microLB } // estimate } // handle_connections() - Waiting::Waiting(tcp_ptr incoming) - : conn(incoming), total(0) + Waiting::Waiting(net::Stream_ptr incoming) + : conn(std::move(incoming)), total(0) { // queue incoming data from clients not yet // assigned to a node @@ -150,7 +152,7 @@ namespace microLB } } } - bool Nodes::assign(tcp_ptr conn, queue_vector_t& readq) + bool Nodes::assign(net::Stream_ptr conn, queue_vector_t& readq) { for (size_t i = 0; i < nodes.size(); i++) { @@ -163,7 +165,7 @@ namespace microLB assert(outgoing->is_connected()); LBOUT("Assigning client to node %d (%s)\n", algo_iterator, outgoing->to_string().c_str()); - this->create_session(not readq.empty(), conn, outgoing); + this->create_session(not readq.empty(), std::move(conn), std::move(outgoing)); // flush readq for (auto buffer : readq) { LBOUT("*** Flushing %lu bytes\n", buffer->size()); @@ -202,15 +204,15 @@ namespace microLB int32_t Nodes::timed_out_sessions() const { return session_timeouts; } - void Nodes::create_session(bool talk, tcp_ptr client, tcp_ptr outgoing) + void Nodes::create_session(bool talk, net::Stream_ptr client, net::Stream_ptr outgoing) { int idx = -1; if (free_sessions.empty()) { idx = sessions.size(); - sessions.emplace_back(*this, idx, talk, client, outgoing); + sessions.emplace_back(*this, idx, talk, std::move(client), std::move(outgoing)); } else { idx = free_sessions.back(); - new (&sessions[idx]) Session(*this, idx, talk, client, outgoing); + new (&sessions[idx]) Session(*this, idx, talk, std::move(client), std::move(outgoing)); free_sessions.pop_back(); } session_total++; @@ -264,7 +266,7 @@ namespace microLB if (conn != nullptr) { // hopefully put this to good use - pool.push_back(conn); + pool.push_back(std::make_unique(conn)); // stop any active check this->stop_active_check(); // signal change in pool @@ -337,7 +339,7 @@ namespace microLB { LBOUT("Connected to %s (%ld total)\n", addr.to_string().c_str(), pool.size()); - this->pool.push_back(conn); + this->pool.push_back(std::make_unique(conn)); // stop any active check this->stop_active_check(); } @@ -348,10 +350,10 @@ namespace microLB this->pool_signal(); }); } - tcp_ptr Node::get_connection() + net::Stream_ptr Node::get_connection() { while (pool.empty() == false) { - auto conn = pool.back(); + auto conn = std::move(pool.back()); assert(conn != nullptr); pool.pop_back(); if (conn->is_connected()) return conn; @@ -361,8 +363,10 @@ namespace microLB } // use indexing to access Session because std::vector - Session::Session(Nodes& n, int idx, bool talk, tcp_ptr inc, tcp_ptr out) - : parent(n), self(idx), incoming(inc), outgoing(out) + Session::Session(Nodes& n, int idx, bool talk, + net::Stream_ptr inc, net::Stream_ptr out) + : parent(n), self(idx), incoming(std::move(inc)), + outgoing(std::move(out)) { // if the client talked before it was assigned a session, use bigger timeout auto timeout = (talk) ? ROLLING_SESSION_TIMEOUT : INITIAL_SESSION_TIMEOUT; @@ -377,10 +381,10 @@ namespace microLB this->handle_timeout(); this->outgoing->write(buf); }); - incoming->on_disconnect( - [&nodes = n, idx] (auto conn, auto /*reason*/) mutable { + incoming->on_close( + [&nodes = n, idx] () { nodes.get_session(idx).outgoing->close(); - conn->close(); + nodes.get_session(idx).incoming->close(); }); outgoing->on_read(READQ_FOR_NODES, [this] (auto buf) { @@ -388,10 +392,10 @@ namespace microLB this->handle_timeout(); this->incoming->write(buf); }); - outgoing->on_disconnect( - [&nodes = n, idx] (auto conn, auto /*reason*/) mutable { + outgoing->on_close( + [&nodes = n, idx] () { + nodes.get_session(idx).outgoing->close(); nodes.get_session(idx).incoming->close(); - conn->close(); }); outgoing->on_close( [&nodes = n, idx] () { diff --git a/lib/microLB/micro_lb/balancer.hpp b/lib/microLB/micro_lb/balancer.hpp index 7eda3889cf..e2d6c01299 100644 --- a/lib/microLB/micro_lb/balancer.hpp +++ b/lib/microLB/micro_lb/balancer.hpp @@ -10,28 +10,28 @@ namespace microLB typedef delegate pool_signal_t; struct Waiting { - Waiting(tcp_ptr); + Waiting(net::Stream_ptr); Waiting(liu::Restore&, net::TCP&); void serialize(liu::Storage&); - tcp_ptr conn; + net::Stream_ptr conn; queue_vector_t readq; int total = 0; }; struct Nodes; struct Session { - Session(Nodes&, int idx, bool talk, tcp_ptr inc, tcp_ptr out); + Session(Nodes&, int idx, bool talk, net::Stream_ptr in, net::Stream_ptr out); bool is_alive() const; void handle_timeout(); void timeout(Nodes&); void serialize(liu::Storage&); - Nodes& parent; - const int self; - int timeout_timer; - tcp_ptr incoming; - tcp_ptr outgoing; + Nodes& parent; + const int self; + int timeout_timer; + net::Stream_ptr incoming; + net::Stream_ptr outgoing; }; struct Node { @@ -46,13 +46,13 @@ namespace microLB void perform_active_check(); void stop_active_check(); void connect(); - tcp_ptr get_connection(); + net::Stream_ptr get_connection(); netstack_t& stack; private: net::Socket addr; const pool_signal_t& pool_signal; - std::vector pool; + std::vector pool; bool active = false; int active_timer = -1; signed int connecting = 0; @@ -77,8 +77,8 @@ namespace microLB template void add_node(Args&&... args); void create_connections(int total); - bool assign(tcp_ptr, queue_vector_t&); - void create_session(bool talk, tcp_ptr inc, tcp_ptr out); + bool assign(net::Stream_ptr, queue_vector_t&); + void create_session(bool talk, net::Stream_ptr inc, net::Stream_ptr out); void close_session(int, bool timeout = false); Session& get_session(int); @@ -112,7 +112,7 @@ namespace microLB const pool_signal_t& get_pool_signal() const; private: - void incoming(tcp_ptr); + void incoming(net::Stream_ptr); void handle_connections(); void handle_queue(); void init_liveupdate(); diff --git a/lib/microLB/micro_lb/serialize.cpp b/lib/microLB/micro_lb/serialize.cpp index 3a4c9aaa81..841c841e6e 100644 --- a/lib/microLB/micro_lb/serialize.cpp +++ b/lib/microLB/micro_lb/serialize.cpp @@ -1,7 +1,7 @@ #include "balancer.hpp" #include -#define LB_VERBOSE 0 +#define LB_VERBOSE 1 #if LB_VERBOSE #define LBOUT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else @@ -39,8 +39,8 @@ namespace microLB void Session::serialize(Storage& store) { - store.add_connection(120, incoming); - store.add_connection(121, outgoing); + //store.add_connection(120, incoming); + //store.add_connection(121, outgoing); store.put_marker(120); } @@ -61,32 +61,34 @@ namespace microLB LBOUT("Deserialize %llu sessions\n", tot_sessions); for(auto i = 0; i < static_cast(tot_sessions); i++) { - auto incoming = store.as_tcp_connection(tcp_in); store.go_next(); - auto outgoing = store.as_tcp_connection(tcp_out); store.go_next(); + //auto incoming = store.as_tcp_connection(tcp_in); store.go_next(); + //auto outgoing = store.as_tcp_connection(tcp_out); store.go_next(); store.pop_marker(120); - create_session(true /* no readq atm */, incoming, outgoing); + //create_session(true /* no readq atm */, incoming, outgoing); } } void Waiting::serialize(liu::Storage& store) { + /* store.add_connection(10, this->conn); store.add_int(11, (int) readq.size()); for (auto buffer : readq) { store.add_buffer(12, buffer->data(), buffer->size()); - } + }*/ store.put_marker(10); } Waiting::Waiting(liu::Restore& store, net::TCP& stack) { + /* this->conn = store.as_tcp_connection(stack); store.go_next(); int qsize = store.as_int(); store.go_next(); for (int i = 0; i < qsize; i++) { auto buf = store.as_buffer(); store.go_next(); readq.push_back(net::tcp::construct_buffer(buf.begin(), buf.end())); - } + }*/ store.pop_marker(10); } From bb04ea710bb10a0b9bb855295bf398d711367be4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 17 Apr 2018 17:05:18 +0200 Subject: [PATCH 195/723] posix: avoid FD_map reusing the same id when template arg differs --- api/posix/fd_map.hpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/api/posix/fd_map.hpp b/api/posix/fd_map.hpp index 4adb95e802..dcf51de04c 100644 --- a/api/posix/fd_map.hpp +++ b/api/posix/fd_map.hpp @@ -68,14 +68,19 @@ class FD_map { { instance().internal_close(id); } private: + std::map> map_; + id_t counter_ = 5; + + FD_map() = default; + + auto& counter() noexcept + { return counter_; } + void internal_close(const id_t id) { auto erased = map_.erase(id); assert(erased > 0); } - - std::map> map_; - FD_map() {} }; template @@ -84,8 +89,8 @@ T& FD_map::open(Args&&... args) static_assert(std::is_base_of::value, "Template argument is not a File Descriptor (FD)"); - static id_t counter = 5; - const auto id = counter++; + + const auto id = counter()++; auto* fd = new T(id, std::forward(args)...); From 0967fcdfbe1e848da10ca7f61a1c34bbd0b0c2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 17 Apr 2018 17:27:43 +0200 Subject: [PATCH 196/723] test: verify vfs plugin in conf test --- test/posix/integration/conf/config.json | 2 +- test/posix/integration/conf/disk/{ => files}/passwd | 0 test/posix/integration/conf/test_pathconf.c | 2 ++ 3 files changed, 3 insertions(+), 1 deletion(-) rename test/posix/integration/conf/disk/{ => files}/passwd (100%) diff --git a/test/posix/integration/conf/config.json b/test/posix/integration/conf/config.json index 59bdf27afd..5a70abf6b5 100644 --- a/test/posix/integration/conf/config.json +++ b/test/posix/integration/conf/config.json @@ -2,7 +2,7 @@ "vfs" : [ { "disk" : "memdisk", - "root" : "/", + "root" : "/files", "mount" : "/etc", "description" : "test fs" } diff --git a/test/posix/integration/conf/disk/passwd b/test/posix/integration/conf/disk/files/passwd similarity index 100% rename from test/posix/integration/conf/disk/passwd rename to test/posix/integration/conf/disk/files/passwd diff --git a/test/posix/integration/conf/test_pathconf.c b/test/posix/integration/conf/test_pathconf.c index 4b2122968b..d7d0d77153 100644 --- a/test/posix/integration/conf/test_pathconf.c +++ b/test/posix/integration/conf/test_pathconf.c @@ -21,6 +21,7 @@ #include #include #include +#include #define PCV(x) print_conf_value(x, #x) @@ -29,6 +30,7 @@ const char* filename = "/etc/passwd"; static void print_conf_value(int name, const char* sym) { errno = 0; int fd = open(filename, O_RDONLY); + assert(fd > 0); if (fd == -1) { printf("Couldn't open %s\n", filename); exit(1); From 5a8c6e366e1394b4d17cd6177526cdd9767bbc4a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 18 Apr 2018 13:53:47 +0200 Subject: [PATCH 197/723] microLB: Fix session creation issue --- lib/microLB/micro_lb/balancer.cpp | 14 ++++++++------ lib/microLB/micro_lb/balancer.hpp | 4 ++-- lib/microLB/micro_lb/serialize.cpp | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/lib/microLB/micro_lb/balancer.cpp b/lib/microLB/micro_lb/balancer.cpp index 08b889b57d..26f5d98b0e 100644 --- a/lib/microLB/micro_lb/balancer.cpp +++ b/lib/microLB/micro_lb/balancer.cpp @@ -13,7 +13,7 @@ #define INITIAL_SESSION_TIMEOUT 5s #define ROLLING_SESSION_TIMEOUT 60s -#define LB_VERBOSE 1 +#define LB_VERBOSE 0 #if LB_VERBOSE #define LBOUT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else @@ -50,7 +50,7 @@ namespace microLB } netstack_t& Balancer::get_nodes_network() noexcept { - return this->netin; + return this->netout; } const pool_signal_t& Balancer::get_pool_signal() const { @@ -165,11 +165,12 @@ namespace microLB assert(outgoing->is_connected()); LBOUT("Assigning client to node %d (%s)\n", algo_iterator, outgoing->to_string().c_str()); - this->create_session(not readq.empty(), std::move(conn), std::move(outgoing)); - // flush readq + auto& session = this->create_session( + not readq.empty(), std::move(conn), std::move(outgoing)); + // flush readq to session.outgoing for (auto buffer : readq) { LBOUT("*** Flushing %lu bytes\n", buffer->size()); - outgoing->write(buffer); + session.outgoing->write(buffer); } return true; } @@ -204,7 +205,7 @@ namespace microLB int32_t Nodes::timed_out_sessions() const { return session_timeouts; } - void Nodes::create_session(bool talk, net::Stream_ptr client, net::Stream_ptr outgoing) + Session& Nodes::create_session(bool talk, net::Stream_ptr client, net::Stream_ptr outgoing) { int idx = -1; if (free_sessions.empty()) { @@ -219,6 +220,7 @@ namespace microLB session_cnt++; LBOUT("New session %d (current = %d, total = %ld)\n", idx, session_cnt, session_total); + return sessions[idx]; } Session& Nodes::get_session(int idx) { diff --git a/lib/microLB/micro_lb/balancer.hpp b/lib/microLB/micro_lb/balancer.hpp index e2d6c01299..a86155972c 100644 --- a/lib/microLB/micro_lb/balancer.hpp +++ b/lib/microLB/micro_lb/balancer.hpp @@ -78,8 +78,8 @@ namespace microLB void add_node(Args&&... args); void create_connections(int total); bool assign(net::Stream_ptr, queue_vector_t&); - void create_session(bool talk, net::Stream_ptr inc, net::Stream_ptr out); - void close_session(int, bool timeout = false); + Session& create_session(bool talk, net::Stream_ptr inc, net::Stream_ptr out); + void close_session(int, bool timeout = false); Session& get_session(int); void serialize(liu::Storage&); diff --git a/lib/microLB/micro_lb/serialize.cpp b/lib/microLB/micro_lb/serialize.cpp index 841c841e6e..674b7d496e 100644 --- a/lib/microLB/micro_lb/serialize.cpp +++ b/lib/microLB/micro_lb/serialize.cpp @@ -1,7 +1,7 @@ #include "balancer.hpp" #include -#define LB_VERBOSE 1 +#define LB_VERBOSE 0 #if LB_VERBOSE #define LBOUT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else From 3d306f1f3935f441487539856eb397b32d90c86a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 19 Apr 2018 11:14:49 +0200 Subject: [PATCH 198/723] musl: Added UDP syslog unix backend --- api/posix/syslog_print_socket.hpp | 10 +++- api/posix/syslog_udp_socket.hpp | 86 +++++++++++++++++++++++++++++++ api/posix/unix_fd.hpp | 6 ++- api/posix/unix_fd_impl.hpp | 5 +- src/posix/unix_fd.cpp | 13 +++-- 5 files changed, 112 insertions(+), 8 deletions(-) create mode 100644 api/posix/syslog_udp_socket.hpp diff --git a/api/posix/syslog_print_socket.hpp b/api/posix/syslog_print_socket.hpp index cb9c4f97e2..233eed82ef 100644 --- a/api/posix/syslog_print_socket.hpp +++ b/api/posix/syslog_print_socket.hpp @@ -22,12 +22,20 @@ class Syslog_print_socket : public Unix_FD_impl { public: + Syslog_print_socket() = default; + inline long connect(const struct sockaddr *, socklen_t) override; inline ssize_t sendto(const void* buf, size_t, int fl, const struct sockaddr* addr, socklen_t addrlen) override; - + inline int close() override { return 0; } ~Syslog_print_socket() = default; }; +long Syslog_print_socket::connect(const struct sockaddr *, socklen_t) +{ + // connect doesnt really mean anything here + return 0; +} + #include ssize_t Syslog_print_socket::sendto(const void* buf, size_t len, int /*fl*/, const struct sockaddr* /*addr*/, diff --git a/api/posix/syslog_udp_socket.hpp b/api/posix/syslog_udp_socket.hpp new file mode 100644 index 0000000000..1a38ab89df --- /dev/null +++ b/api/posix/syslog_udp_socket.hpp @@ -0,0 +1,86 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef POSIX_SYSLOG_UDP_SOCKET_HPP +#define POSIX_SYSLOG_UDP_SOCKET_HPP + +#include +#include +#include + +class Syslog_UDP_socket : public Unix_FD_impl { +public: + inline Syslog_UDP_socket(net::Inet& net, + const net::ip4::Addr raddr, const uint16_t rport); + + inline long connect(const struct sockaddr *, socklen_t) override; + inline ssize_t sendto(const void* buf, size_t, int fl, + const struct sockaddr* addr, socklen_t addrlen) override; + inline int close() override; + + inline ~Syslog_UDP_socket(); + +private: + net::Inet& stack; + net::UDPSocket* udp; + const net::ip4::Addr addr; + const uint16_t port; +}; + +Syslog_UDP_socket::Syslog_UDP_socket(net::Inet& net, + const net::ip4::Addr raddr, + const uint16_t rport) + : stack{net}, udp{nullptr}, addr{raddr}, port{rport} +{ +} + +long Syslog_UDP_socket::connect(const struct sockaddr*, socklen_t) +{ + // so we can use the call to connect to bind to a UDP socket. + // we can also hopefully assume that the network is configured + // and bind wont cause any issues. + Expects(stack.is_configured()); + if(udp == nullptr) + udp = &stack.udp().bind(); + return 0; +} + +ssize_t Syslog_UDP_socket::sendto(const void* buf, size_t len, int /*fl*/, + const struct sockaddr*, socklen_t) +{ + Expects(udp != nullptr); + udp->sendto(addr, port, buf, len); + return len; +} + +int Syslog_UDP_socket::close() +{ + if(udp) + { + udp->close(); + udp = nullptr; + } + return 0; +} + +Syslog_UDP_socket::~Syslog_UDP_socket() +{ + if(udp != nullptr) + udp->close(); +} + +#endif diff --git a/api/posix/unix_fd.hpp b/api/posix/unix_fd.hpp index 0a5ec35e7d..969cf28a91 100644 --- a/api/posix/unix_fd.hpp +++ b/api/posix/unix_fd.hpp @@ -27,7 +27,8 @@ class Unix_FD : public SockFD { using Impl = Unix_FD_impl; Unix_FD(int id, int type) : SockFD(id), type_(type) - {} + { + } /** SOCKET */ long connect(const struct sockaddr *, socklen_t) override; @@ -36,7 +37,8 @@ class Unix_FD : public SockFD { int close() override; private: Impl* impl = nullptr; - const int type_; + const int type_ [[maybe_unused]]; // it's probably gonna be necessary + // to tell if socket is stream or dgram long set_impl_if_needed(const struct sockaddr* addr, socklen_t addrlen); }; diff --git a/api/posix/unix_fd_impl.hpp b/api/posix/unix_fd_impl.hpp index df20ba8ec7..0f0bec3995 100644 --- a/api/posix/unix_fd_impl.hpp +++ b/api/posix/unix_fd_impl.hpp @@ -21,9 +21,10 @@ #include class Unix_FD_impl { public: - //virtual long connect(const struct sockaddr *, socklen_t) = 0; + virtual long connect(const struct sockaddr *, socklen_t) = 0; virtual ssize_t sendto(const void* buf, size_t, int fl, - const struct sockaddr* addr, socklen_t addrlen) = 0; + const struct sockaddr* addr, socklen_t addrlen) = 0; + virtual int close() = 0; virtual ~Unix_FD_impl() = default; }; diff --git a/src/posix/unix_fd.cpp b/src/posix/unix_fd.cpp index 448cbcee8e..c93716a4ea 100644 --- a/src/posix/unix_fd.cpp +++ b/src/posix/unix_fd.cpp @@ -38,15 +38,16 @@ long Unix_FD::set_impl_if_needed(const struct sockaddr* addr, socklen_t addrlen) impl = &ent; return 0; } - catch (const fs::Err_not_found& e) { - printf("no ent: %s\n", e.what()); + catch (const fs::Err_not_found& /*e*/) { + //printf("no ent: %s\n", e.what()); return -ENOENT; } } long Unix_FD::connect(const struct sockaddr* addr, socklen_t addrlen) { - return set_impl_if_needed(addr, addrlen); + auto res = set_impl_if_needed(addr, addrlen); + return (res < 0) ? res : impl->connect(addr, addrlen); } ssize_t Unix_FD::sendto(const void* buf, size_t len, int fl, @@ -59,5 +60,11 @@ ssize_t Unix_FD::sendto(const void* buf, size_t len, int fl, int Unix_FD::close() { + if(impl) + { + auto res = impl->close(); + impl = nullptr; + return res; + } return 0; } From 7c10f965a05de023824668860b4f75c2fbb6eb68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 19 Apr 2018 11:15:57 +0200 Subject: [PATCH 199/723] plugin: Added syslog unix backend plugin --- src/plugins/CMakeLists.txt | 4 ++ src/plugins/syslog.cpp | 113 +++++++++++++++++++++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 src/plugins/syslog.cpp diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index faaceda002..6ec3a8f68c 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -38,6 +38,9 @@ add_library(field_medic STATIC field_medic/diag.cpp) add_dependencies(field_medic PrecompiledLibraries) +add_library(syslog STATIC syslog.cpp) +add_dependencies(syslog PrecompiledLibraries) + # # Installation # @@ -54,4 +57,5 @@ install(TARGETS nacl vfs field_medic + syslog DESTINATION includeos/${ARCH}/plugins) diff --git a/src/plugins/syslog.cpp b/src/plugins/syslog.cpp new file mode 100644 index 0000000000..3591c489af --- /dev/null +++ b/src/plugins/syslog.cpp @@ -0,0 +1,113 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This file contains a plugin for automatically mounting +// a given syslog implementation to the virtual filesystem (VFS) + +#include +#include +#include +#include +#include + +#ifndef RAPIDJSON_HAS_STDSTRING + #define RAPIDJSON_HAS_STDSTRING 1 +#endif + +#include +#include + +static std::unique_ptr syslog_impl = nullptr; +const static std::string default_path{"/dev/log"}; +const static uint16_t default_port{6514}; + +static void mount_print_sock(const std::string& path) +{ + INFO("Syslog", "Mounting Syslog Print backend on %s", path.c_str()); + syslog_impl.reset(new Syslog_print_socket()); + fs::mount(path, *syslog_impl, "Syslog Print Unix Backend"); +} + +static void mount_udp_sock(const std::string& path, net::Inet& stack, + const net::ip4::Addr addr, const uint16_t port) +{ + INFO("Syslog", "Mounting Syslog UDP backend on %s", path.c_str()); + INFO2("%s:%u @ %s", addr.to_string().c_str(), port, stack.ifname().c_str()); + syslog_impl.reset(new Syslog_UDP_socket(stack, addr, port)); + fs::mount(path, *syslog_impl, "Syslog UDP Unix Backend"); +} + +static void mount_default_sock() +{ + mount_print_sock(default_path); +} + +// Function being run by the OS for mounting resources +static void syslog_mount() +{ + const auto& json = ::Config::get(); + + // No config, use default + if(json.empty()) + { + mount_default_sock(); + return; + } + + using namespace rapidjson; + Document doc; + doc.Parse(json.data()); + + Expects(doc.IsObject() && "Malformed config"); + + // No syslog member, use default + if(not doc.HasMember("syslog")) + { + mount_default_sock(); + return; + } + + const auto& cfg = doc["syslog"]; + + // get the path if any, else use default + const std::string path = (cfg.HasMember("path")) + ? cfg["path"].GetString() : default_path; + + // if no type are given, use print socket + if(not cfg.HasMember("type") or not (cfg["type"].GetString() == std::string{"udp"})) + { + mount_print_sock(path); + return; + } + + // check type (only UDP for now) + Expects(cfg.HasMember("iface") && "Missing iface (index)"); + Expects(cfg.HasMember("address") && "Missing address"); + + auto& stack = net::Super_stack::get(cfg["iface"].GetInt()); + const net::ip4::Addr addr{cfg["address"].GetString()}; + const uint16_t port = cfg.HasMember("port") ? + cfg["port"].GetUint() : default_port; + + mount_udp_sock(path, stack, addr, port); +} + +#include +__attribute__((constructor)) +static void syslog_plugin() { + OS::register_plugin(syslog_mount, "Syslog Unix backend"); +} From 298f2e6996d6bf4eea707840a62b24485ec12021 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 19 Apr 2018 11:18:04 +0200 Subject: [PATCH 200/723] test: Changes to the syslog tests, have them using syslog plugin --- .../integration/syslog_default/CMakeLists.txt | 5 +- .../integration/syslog_default/service.cpp | 5 -- .../integration/syslog_plugin/CMakeLists.txt | 2 +- .../integration/syslog_plugin/config.json | 9 ++++ .../integration/syslog_plugin/service.cpp | 52 +++---------------- test/posix/integration/syslog_plugin/test.py | 37 +++++++------ 6 files changed, 40 insertions(+), 70 deletions(-) create mode 100644 test/posix/integration/syslog_plugin/config.json diff --git a/test/posix/integration/syslog_default/CMakeLists.txt b/test/posix/integration/syslog_default/CMakeLists.txt index 1c737cd75e..dc92be704b 100644 --- a/test/posix/integration/syslog_default/CMakeLists.txt +++ b/test/posix/integration/syslog_default/CMakeLists.txt @@ -14,14 +14,11 @@ set(SERVICE_NAME "Syslog Default Test Service") # Name of your service binary set(BINARY "test_syslog_default") -# Maximum memory can be hard-coded into the binary -set(MAX_MEM 128) - - # Source files to be linked with OS library parts to form bootable image set(SOURCES service.cpp ) +set(PLUGINS syslog) # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/posix/integration/syslog_default/service.cpp b/test/posix/integration/syslog_default/service.cpp index 613429f8ea..e2cf350460 100644 --- a/test/posix/integration/syslog_default/service.cpp +++ b/test/posix/integration/syslog_default/service.cpp @@ -22,14 +22,9 @@ /* For testing IncludeOS */ #include -#include #include int main() { - static Syslog_print_socket syslog_sock_impl; - const fs::Path path{"/dev/log"}; - printf("path: %s\n", path.to_string().c_str()); - fs::VFS::mount(path, syslog_sock_impl, "Just a printing syslog"); /* ----------- Integration test for the default syslog behavior: printf ----------- */ /* ------------------------- Testing POSIX syslog ------------------------- */ diff --git a/test/posix/integration/syslog_plugin/CMakeLists.txt b/test/posix/integration/syslog_plugin/CMakeLists.txt index 9c3d2ca917..6e6a03cd92 100644 --- a/test/posix/integration/syslog_plugin/CMakeLists.txt +++ b/test/posix/integration/syslog_plugin/CMakeLists.txt @@ -23,7 +23,7 @@ set(SOURCES service.cpp ) -set(PLUGINS syslogd) +set(PLUGINS syslogd syslog) set(DRIVERS virtionet) # include service build script diff --git a/test/posix/integration/syslog_plugin/config.json b/test/posix/integration/syslog_plugin/config.json new file mode 100644 index 0000000000..40fca26f54 --- /dev/null +++ b/test/posix/integration/syslog_plugin/config.json @@ -0,0 +1,9 @@ +{ + "syslog": { + "type": "udp", + "path": "/dev/log", + "iface": 0, + "address": "10.0.0.2", + "port": 6514 + } +} diff --git a/test/posix/integration/syslog_plugin/service.cpp b/test/posix/integration/syslog_plugin/service.cpp index 3fde3b2017..2cb1d8861e 100644 --- a/test/posix/integration/syslog_plugin/service.cpp +++ b/test/posix/integration/syslog_plugin/service.cpp @@ -17,9 +17,6 @@ #include -/* For testing Posix */ -#include - /* For testing IncludeOS */ #include @@ -35,60 +32,25 @@ int main() auto& inet = net::Inet4::stack(); // static IP in case DHCP fails inet.network_config({ 10, 0, 0, 47 }, // IP - { 255, 255, 255, 0 }, // Netmask - { 10, 0, 0, 1 }, // Gateway - { 10, 0, 0, 1} ); // DNS - - // Setting IP and port for the syslog messages - Syslog::settings( {10, 0, 0, 2}, 6514 ); - printf("Syslog messages are sent to IP %s and port %d\n", Syslog::ip().str().c_str(), Syslog::port()); + { 255, 255, 255, 0 }, // Netmask + { 10, 0, 0, 1 }, // Gateway + { 10, 0, 0, 1} ); // DNS // Starts the python integration test: printf("Service IP address is %s\n", inet.ip_addr().str().c_str()); int invalid_priority = -1; - syslog(invalid_priority, "Invalid %d", invalid_priority); - - invalid_priority = 10; - syslog(invalid_priority, "Invalid %d", invalid_priority); - - invalid_priority = 55; - syslog(invalid_priority, "Invalid %d", invalid_priority); std::string one{"one"}; std::string two{"two"}; size_t number = 33; - syslog(LOG_INFO, "(Info) No open has been called prior to this"); - syslog(LOG_NOTICE, "(Notice) Program created with two arguments: %s and %s", one.c_str(), two.c_str()); - - openlog("Prepended message", LOG_CONS | LOG_NDELAY, LOG_MAIL); - - syslog(LOG_ERR, "(Err) Log after prepended message with one argument: %d", 44); - syslog(LOG_WARNING, "(Warning) Log number two after openlog set prepended message"); - - closelog(); - syslog(LOG_WARNING, "(Warning) Log after closelog with three arguments. One is %u, another is %s, a third is %d", number, "this", 4011); - - openlog("Second prepended message", LOG_PID | LOG_CONS, LOG_USER); - - syslog(LOG_EMERG, "Emergency log after openlog and new facility: user"); - syslog(LOG_ALERT, "Alert log with the m argument: %m"); - - closelog(); - - syslog(LOG_CRIT, "Critical after cleared prepended message (closelog has been called)"); - - closelog(); - - openlog("Open after close prepended message", LOG_CONS, LOG_MAIL); - - syslog(LOG_INFO, "Info after openlog with both m: %m and two hex arguments: 0x%x and 0x%x", 100, 50); - - closelog(); /* ------------------------- Testing IncludeOS syslog ------------------------- */ + // Setting IP and port for the syslog messages + Syslog::settings( {10, 0, 0, 2}, 6514 ); + printf("Syslog messages are sent to IP %s and port %d\n", Syslog::ip().str().c_str(), Syslog::port()); invalid_priority = -1; Syslog::syslog(invalid_priority, "Invalid %d", invalid_priority); @@ -109,7 +71,7 @@ int main() Syslog::closelog(); - Syslog::syslog(LOG_WARNING, "(Warning) Log after closelog with three arguments. One is %u, another is %s, a third is %d", number, "this", 4011); + Syslog::syslog(LOG_WARNING, "(Warning) Log after closelog with three arguments. One is %zu, another is %s, a third is %d", number, "this", 4011); Syslog::openlog("Second prepended message", LOG_PID | LOG_CONS, LOG_USER); diff --git a/test/posix/integration/syslog_plugin/test.py b/test/posix/integration/syslog_plugin/test.py index c978aeff38..394d95ae09 100755 --- a/test/posix/integration/syslog_plugin/test.py +++ b/test/posix/integration/syslog_plugin/test.py @@ -17,12 +17,19 @@ # Gateway IP is 10.0.0.2 - syslog sends its messages here on port 6514 # Set up a temporary interface -subprocess.call(["sudo", "ifconfig", "bridge43:0", "10.0.0.2/24"]) +import platform +if platform.system() == 'Darwin': + subprocess.call(["sudo", "ifconfig", "bridge43", "alias", "10.0.0.2/24"]) +else: + subprocess.call(["sudo", "ifconfig", "bridge43:0", "10.0.0.2/24"]) # Tear down interface on exit @atexit.register def tear_down(): - subprocess.call(["sudo", "ifconfig", "bridge43:0", "down"]) + if platform.system() == 'Darwin': + subprocess.call(["sudo", "ifconfig", "bridge43", "-alias", "10.0.0.2"]) + else: + subprocess.call(["sudo", "ifconfig", "bridge43:0", "down"]) UDP_IP = "10.0.0.2" UDP_PORT = 6514 @@ -31,46 +38,46 @@ def tear_down(): sock.bind((UDP_IP, UDP_PORT)) num_received = 0 -num_expected_msgs = 24 +num_expected_msgs = 12 -pre_msg1 = "<11>1 " +pre_msg1 = "<67>1 " post_msg1 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - Syslog: Unknown priority -1. Message: Invalid -1" -pre_msg2 = "<11>1 " +pre_msg2 = "<67>1 " post_msg2 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - Syslog: Unknown priority 10. Message: Invalid 10" -pre_msg3 = "<11>1 " +pre_msg3 = "<67>1 " post_msg3 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - Syslog: Unknown priority 55. Message: Invalid 55" -pre_msg4 = "<14>1 " +pre_msg4 = "<70>1 " post_msg4 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - (Info) No open has been called prior to this" -pre_msg5 = "<13>1 " +pre_msg5 = "<69>1 " post_msg5 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - (Notice) Program created with two arguments: one and two" -pre_msg6 = "<19>1 " +pre_msg6 = "<131>1 " post_msg6 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - Prepended message (Err) Log after prepended message with " + \ "one argument: 44" -pre_msg7 = "<20>1 " +pre_msg7 = "<132>1 " post_msg7 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - Prepended message (Warning) Log number two after openlog " + \ "set prepended message" -pre_msg8 = "<12>1 " +pre_msg8 = "<68>1 " post_msg8 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - (Warning) Log after closelog with three arguments. One is 33, " + \ "another is this, a third is 4011" -pre_msg9 = "<8>1 " +pre_msg9 = "<64>1 " post_msg9 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - Second prepended message Emergency log after openlog and new " + \ "facility: user" -pre_msg10 = "<9>1 " +pre_msg10 = "<65>1 " post_msg10 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - Second prepended message Alert log with the m argument: Success" -pre_msg11 = "<10>1 " +pre_msg11 = "<66>1 " post_msg11 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - Critical after cleared prepended message (closelog has been called)" -pre_msg12 = "<22>1 " +pre_msg12 = "<134>1 " post_msg12 = " 10.0.0.47 test_syslog_plugin 1 UDPOUT - Open after close prepended message Info after openlog with " + \ "both m: Success and two hex arguments: 0x64 and 0x32" From edcfbfc8badeb86abcc36ff5360cff310256037f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 19 Apr 2018 13:29:24 +0200 Subject: [PATCH 201/723] microLB: Add support for OpenSSL TLS-termination --- api/net/openssl/init.hpp | 2 + lib/microLB/CMakeLists.txt | 2 + lib/microLB/micro_lb/autoconf.cpp | 16 ++++- lib/microLB/micro_lb/balancer.hpp | 3 + lib/microLB/micro_lb/openssl.cpp | 47 ++++++++++++ src/CMakeLists.txt | 6 +- src/net/https/openssl_server.cpp | 74 +------------------ src/net/openssl/server.cpp | 80 +++++++++++++++++++++ test/net/integration/microLB/CMakeLists.txt | 2 + test/net/integration/microLB/config.json | 6 +- test/net/integration/microLB/drive/test.key | 52 ++++++++++++++ test/net/integration/microLB/drive/test.pem | 31 ++++++++ test/net/integration/microLB/test.py | 4 +- 13 files changed, 243 insertions(+), 82 deletions(-) create mode 100644 lib/microLB/micro_lb/openssl.cpp create mode 100644 src/net/openssl/server.cpp create mode 100644 test/net/integration/microLB/drive/test.key create mode 100644 test/net/integration/microLB/drive/test.pem diff --git a/api/net/openssl/init.hpp b/api/net/openssl/init.hpp index 71547d0cf2..393363544b 100644 --- a/api/net/openssl/init.hpp +++ b/api/net/openssl/init.hpp @@ -8,6 +8,8 @@ namespace openssl extern void verify_rng(); extern void init(); + extern SSL_CTX* create_server(const std::string& cert, const std::string& key); + extern SSL_CTX* create_client(fs::List, bool verify_peer = false); // enable peer certificate verification extern void client_verify_peer(SSL_CTX*); diff --git a/lib/microLB/CMakeLists.txt b/lib/microLB/CMakeLists.txt index 76982ffda7..007d3ffebc 100644 --- a/lib/microLB/CMakeLists.txt +++ b/lib/microLB/CMakeLists.txt @@ -7,6 +7,7 @@ add_definitions(-DARCH="${ARCH}") add_library(microlb STATIC micro_lb/autoconf.cpp micro_lb/balancer.cpp + micro_lb/openssl.cpp micro_lb/serialize.cpp ) @@ -20,6 +21,7 @@ target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/api) target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/lib/LiveUpdate) target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/mod/rapidjson/include) target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/mod/GSL/) +target_include_directories(microlb PUBLIC ${OPENSSL_DIR}/include) install(TARGETS microlb DESTINATION includeos/${ARCH}/lib) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/micro_lb/balancer.hpp DESTINATION includeos/include/micro_lb) diff --git a/lib/microLB/micro_lb/autoconf.cpp b/lib/microLB/micro_lb/autoconf.cpp index c0016bfb41..62d33027bc 100644 --- a/lib/microLB/micro_lb/autoconf.cpp +++ b/lib/microLB/micro_lb/autoconf.cpp @@ -33,8 +33,20 @@ namespace microLB auto& netout = net::Super_stack::get(NODE_NET); netout.tcp().set_MSL(15s); - // create load balancer - auto* balancer = new Balancer(netinc, CLIENT_PORT, netout); + Balancer* balancer = nullptr; + + if (clients.HasMember("certificate")) + { + assert(clients.HasMember("key") && "TLS-enabled microLB must also have key"); + // create TLS over TCP load balancer + balancer = new Balancer(netinc, CLIENT_PORT, netout, + clients["certificate"].GetString(), + clients["key"].GetString()); + } + else { + // create TCP load balancer + balancer = new Balancer(netinc, CLIENT_PORT, netout); + } auto& nodelist = nodes["list"]; assert(nodelist.IsArray()); diff --git a/lib/microLB/micro_lb/balancer.hpp b/lib/microLB/micro_lb/balancer.hpp index a86155972c..0deed6bb20 100644 --- a/lib/microLB/micro_lb/balancer.hpp +++ b/lib/microLB/micro_lb/balancer.hpp @@ -98,6 +98,8 @@ namespace microLB struct Balancer { Balancer(netstack_t& in, uint16_t port, netstack_t& out); + Balancer(netstack_t& in, uint16_t port, netstack_t& out, + const std::string& cert, const std::string& key); static Balancer* from_config(); int wait_queue() const; @@ -125,6 +127,7 @@ namespace microLB std::deque queue; int throw_retry_timer = -1; int throw_counter = 0; + void* openssl_data = nullptr; }; template diff --git a/lib/microLB/micro_lb/openssl.cpp b/lib/microLB/micro_lb/openssl.cpp new file mode 100644 index 0000000000..2d0b4ee167 --- /dev/null +++ b/lib/microLB/micro_lb/openssl.cpp @@ -0,0 +1,47 @@ +#include "balancer.hpp" +#include +#include +#include + +namespace microLB +{ + Balancer::Balancer( + netstack_t& in, + uint16_t port, + netstack_t& out, + const std::string& tls_cert, + const std::string& tls_key) + : nodes(), netin(in), netout(out), signal({this, &Balancer::handle_queue}) + { + fs::memdisk().init_fs( + [] (auto err, auto&) { + assert(!err); + }); + + openssl::init(); + openssl::verify_rng(); + + this->openssl_data = openssl::create_server(tls_cert, tls_key); + + netin.tcp().listen(port, + [this] (net::tcp::Connection_ptr conn) { + if (conn != nullptr) + { + auto* stream = new openssl::TLS_stream( + (SSL_CTX*) this->openssl_data, + std::make_unique(conn) + ); + stream->on_connect( + [this, stream] (auto&) { + this->incoming(std::unique_ptr (stream)); + }); + stream->on_close( + [stream] () { + delete stream; + }); + } + }); + + this->init_liveupdate(); + } +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9896130ae8..3c83c7b415 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -17,11 +17,11 @@ include_directories(${INCLUDEOS_ROOT}/mod/GSL/) include_directories(${INCLUDEOS_ROOT}/mod/rapidjson/include) include_directories(${INCLUDEOS_ROOT}/mod/uzlib/src) # tinf.h for tar include_directories(${BOTAN_DIR}) -include_directories(${OPENSSL_DIR}) include_directories(${OPENSSL_DIR}/include) if(${ARCH} STREQUAL "x86_64") - set(OPENSSL_MODULES "net/openssl/init.cpp" "net/openssl/client.cpp" "net/https/openssl_server.cpp" - "net/http/client.cpp") + set(OPENSSL_MODULES "net/openssl/init.cpp" "net/openssl/client.cpp" + "net/openssl/server.cpp" + "net/https/openssl_server.cpp" "net/http/client.cpp") set(OPENSSL_LIBS openssl_ssl openssl_crypto) endif() set(BOTAN_MODULES "net/https/botan_server.cpp") diff --git a/src/net/https/openssl_server.cpp b/src/net/https/openssl_server.cpp index daafbffa17..6502d34c4f 100644 --- a/src/net/https/openssl_server.cpp +++ b/src/net/https/openssl_server.cpp @@ -2,81 +2,9 @@ #include #include #include -#define LOAD_FROM_MEMDISK namespace http { - // https://gist.github.com/darrenjs/4645f115d10aa4b5cebf57483ec82eca - inline void handle_error(const char* file, int lineno, const char* msg) { - fprintf(stderr, "** %s:%i %s\n", file, lineno, msg); - ERR_print_errors_fp(stderr); - exit(1); - } - #define int_error(msg) handle_error(__FILE__, __LINE__, msg) - - static void - tls_load_from_memory(SSL_CTX* ctx, - fs::Buffer cert_buffer, - fs::Buffer key_buffer) - { - auto* cbio = BIO_new_mem_buf(cert_buffer.data(), cert_buffer.size()); - auto* cert = PEM_read_bio_X509(cbio, NULL, 0, NULL); - assert(cert != NULL); - SSL_CTX_use_certificate(ctx, cert); - BIO_free(cbio); - - auto* kbio = BIO_new_mem_buf(key_buffer.data(), key_buffer.size()); - auto* key = PEM_read_bio_RSAPrivateKey(kbio, NULL, 0, NULL); - assert(key != NULL); - SSL_CTX_use_RSAPrivateKey(ctx, key); - BIO_free(kbio); - } - - SSL_CTX* tls_init_server(const char* cert_file, const char* key_file) - { - /* create the SSL server context */ - auto* ctx = SSL_CTX_new(TLSv1_2_method()); - if (!ctx) throw std::runtime_error("SSL_CTX_new()"); - - int res = SSL_CTX_set_cipher_list(ctx, "AES256-SHA"); - assert(res == 1); - - #ifdef LOAD_FROM_MEMDISK - auto& filesys = fs::memdisk().fs(); - // load CA certificate - auto ca_cert_buffer = filesys.read_file(cert_file); - // load CA private key - auto ca_key_buffer = filesys.read_file(key_file); - // use in SSL CTX - tls_load_from_memory(ctx, ca_cert_buffer, ca_key_buffer); - #else - /* Load certificate and private key files, and check consistency */ - int err; - err = SSL_CTX_use_certificate_file(ctx, cert_file, SSL_FILETYPE_PEM); - if (err != 1) - int_error("SSL_CTX_use_certificate_file failed"); - - /* Indicate the key file to be used */ - err = SSL_CTX_use_PrivateKey_file(ctx, key_file, SSL_FILETYPE_PEM); - if (err != 1) - int_error("SSL_CTX_use_PrivateKey_file failed"); - #endif - - /* Make sure the key and certificate file match. */ - if (SSL_CTX_check_private_key(ctx) != 1) - int_error("SSL_CTX_check_private_key failed"); - - /* Recommended to avoid SSLv2 & SSLv3 */ - SSL_CTX_set_options(ctx, SSL_OP_ALL|SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3); - - int error = ERR_get_error(); - if (error) { - printf("Status: %s\n", ERR_error_string(error, nullptr)); - } - assert(error == SSL_ERROR_NONE); - return ctx; - } - void OpenSSL_server::openssl_initialize(const std::string& certif, const std::string& key) { @@ -91,7 +19,7 @@ namespace http /** VERIFY RNG **/ openssl::verify_rng(); - this->m_ctx = tls_init_server(certif.c_str(), key.c_str()); + this->m_ctx = openssl::create_server(certif.c_str(), key.c_str()); assert(ERR_get_error() == 0); } OpenSSL_server::~OpenSSL_server() diff --git a/src/net/openssl/server.cpp b/src/net/openssl/server.cpp new file mode 100644 index 0000000000..7403f7405c --- /dev/null +++ b/src/net/openssl/server.cpp @@ -0,0 +1,80 @@ +#include +#include +#include +#define LOAD_FROM_MEMDISK + +namespace openssl +{ + // https://gist.github.com/darrenjs/4645f115d10aa4b5cebf57483ec82eca + inline static void + handle_error(const char* file, int lineno, const char* msg) { + fprintf(stderr, "** %s:%i %s\n", file, lineno, msg); + ERR_print_errors_fp(stderr); + exit(1); + } + #define int_error(msg) handle_error(__FILE__, __LINE__, msg) + + static void + load_from_memory(SSL_CTX* ctx, + fs::Buffer cert_buffer, + fs::Buffer key_buffer) + { + auto* cbio = BIO_new_mem_buf(cert_buffer.data(), cert_buffer.size()); + auto* cert = PEM_read_bio_X509(cbio, NULL, 0, NULL); + assert(cert != NULL); + SSL_CTX_use_certificate(ctx, cert); + BIO_free(cbio); + + auto* kbio = BIO_new_mem_buf(key_buffer.data(), key_buffer.size()); + auto* key = PEM_read_bio_RSAPrivateKey(kbio, NULL, 0, NULL); + assert(key != NULL); + SSL_CTX_use_RSAPrivateKey(ctx, key); + BIO_free(kbio); + } + + SSL_CTX* create_server(const std::string& cert_file, + const std::string& key_file) + { + /* create the SSL server context */ + auto* ctx = SSL_CTX_new(TLSv1_2_method()); + if (!ctx) throw std::runtime_error("SSL_CTX_new()"); + + int res = SSL_CTX_set_cipher_list(ctx, "AES256-SHA"); + assert(res == 1); + + #ifdef LOAD_FROM_MEMDISK + auto& filesys = fs::memdisk().fs(); + // load CA certificate + auto ca_cert_buffer = filesys.read_file(cert_file); + // load CA private key + auto ca_key_buffer = filesys.read_file(key_file); + // use in SSL CTX + load_from_memory(ctx, ca_cert_buffer, ca_key_buffer); + #else + /* Load certificate and private key files, and check consistency */ + int err; + err = SSL_CTX_use_certificate_file(ctx, cert_file, SSL_FILETYPE_PEM); + if (err != 1) + int_error("SSL_CTX_use_certificate_file failed"); + + /* Indicate the key file to be used */ + err = SSL_CTX_use_PrivateKey_file(ctx, key_file, SSL_FILETYPE_PEM); + if (err != 1) + int_error("SSL_CTX_use_PrivateKey_file failed"); + #endif + + /* Make sure the key and certificate file match. */ + if (SSL_CTX_check_private_key(ctx) != 1) + int_error("SSL_CTX_check_private_key failed"); + + /* Recommended to avoid SSLv2 & SSLv3 */ + SSL_CTX_set_options(ctx, SSL_OP_ALL|SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3); + + int error = ERR_get_error(); + if (error) { + printf("Status: %s\n", ERR_error_string(error, nullptr)); + } + assert(error == SSL_ERROR_NONE); + return ctx; + } +} diff --git a/test/net/integration/microLB/CMakeLists.txt b/test/net/integration/microLB/CMakeLists.txt index 41867cbdef..a5e90c29cd 100644 --- a/test/net/integration/microLB/CMakeLists.txt +++ b/test/net/integration/microLB/CMakeLists.txt @@ -33,3 +33,5 @@ set(LIBRARIES # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) + +diskbuilder(drive) diff --git a/test/net/integration/microLB/config.json b/test/net/integration/microLB/config.json index 02faf8aace..f47fe36025 100644 --- a/test/net/integration/microLB/config.json +++ b/test/net/integration/microLB/config.json @@ -19,9 +19,11 @@ "load_balancer" : { "clients" : { "iface" : 0, - "port" : 80, + "port" : 443, "waitq_limit" : 1000, - "session_limit" : 1000 + "session_limit" : 1000, + "certificate": "/test.pem", + "key": "/test.key" }, "nodes" : { "iface" : 1, diff --git a/test/net/integration/microLB/drive/test.key b/test/net/integration/microLB/drive/test.key new file mode 100644 index 0000000000..0afea2baf1 --- /dev/null +++ b/test/net/integration/microLB/drive/test.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCzNdwPPeuU59Pi +aCSqkmaFYQDWhKZcmX44NaAAzmjNUvNe9u8DEmi+MbmZGYMk5vTNGkPrGM5xZVKs +4urKvtgoWEMrY3uFYyYoqNNpaXrjS5aiBYx2iZDWdDbB0srLOh6liFjfkZVbMNva +gvAb/NV8L6bBWtA54NmA5C0VqfB30dRGKm7CV3wgLyHgmBwHaL31VkahBcTGFFCV +w1QZTNRoTEOhAwb/4rt7YOZkPOOyawCc6B4NUZ3YJ4C7xbJ0t6cv4c/TiofptWDE +RNV2FDkZdh9JLVnJGQtq8Iz9B+0thgKj+JsneYr04KsQ+Ip8EzpqzeqBNFWnzcio +8iBRFxUk9EsQBbSLxKmxe+oDnoznnw1EnTojg8hPvEcDtO9HrAPwYfa7tbj5zkBp +DBPFfYLgZu2D/ARITCVLOB+u4cZo0jIjicX3ubWEXvFVtwdcGm+MUrb7qDMFM0ny +Kw0kvhcVoZHXHUabTc1hCGQVUaI5o4YXpUQv25chtLNdocEN4/Pue89ldjK1ij3c +fwB0z+eShDk9M7H/pr9I7CC1KMxkAbpOiEYj/UfrO+blOgXfc7sH6cOR3ltI79g2 +xW4M3k1gPA24HDaLt1UpNZvgTUhy9MAjespjUMN3XL1WnJhlJC+ydZq2MqJXAA2d +tg2K00zTNK88O58rO9IargmtoeUrSQIDAQABAoICAQClyWmu4AWUV3L4vIdPFOiq +8zYnPcf5WjYeId3HYSwq5lYmwCIUoW8spCuiqqtb9Bz7sRSr5OL2nFmDftjefvbR +O/XHqdyXZUXjz2rk1aPNqhvL/34WGuVWv2P4otzgbP/0+tHc4X1eQzDgUMl32spU +fHCz5yNCp/QO/QeIRxIihobt8ktMlkpKK9AXSiCD6i3xTMNCK2gCJsD2CyE91omZ +gxP9XCOZjVMLrHT2vi2W3M5QWZuTjrGLSeAZ1aZlu5B7B1ePx1Q8rIK8j0E6XzMD +jvcaZ03sb7LUV3zWiAKuXo8Kye4e8p3ONBmNNaBHcDJWo/ARXyzuc7zyLiwfWE9B +uEBJwAC3JNdBdHURP1yeegBgqZEZZfzH8HLZ6dhVQPGbJqqSImKsD2zVhza4GsCh +HqWFLNcrR7xrSuyX0hegvXGIRYjxN9ayFjvIdD5ZIMwpo6FyNJOn/PWOKMRHPqe5 +phTTd/pJrAZ1396XLh0tEVTK2Zl+YAzojfmQH0LIX6Q6loVkpFT7SITfnvSNvnul ++wAGsb+CMvaDicKB3pn8KUtOjBZtOtQJUdY3heCT3CcqMIVunTyQ0fSZFb6ipKKs +/6GjshiC1IBCCBUFWxp6D0JGviFDGZQ1LZpnNatUiWecuU9GbegGVX7RB6OnjR3G +INSyXzKZ9q0f4OThEFGqUQKCAQEA1t6NRL6o8pIhCCSZVuwS1Vx0so0n6WGmvbWa +n4nrx6aAeXO13h9bWUNVNlfPQn9gMHP4iP3IIGm7NP0PL/NpF4AaVnNZeX5SwGFG +LubbmUQBc616ncPdJivZNIFOlqvqxLPxFfG4n/AKeSmeMTEXyxK7OWIipN1vzEiE +uytnkLXSA2Ye4++W+/VwQzTUT3qVhL+RBlM+pOcw1L+wdzRNHcacTSexucgWDMMW +V55SpwUVij8Fvge87am0bBD47Rj80U58cmEFiETw6cCqjmOjqhVHh6VCPdIY3yJd +95oeD7wuvm4CBo+YdhNzOEO2NHtv8YYF+dfGlzg4IKn20MOGqwKCAQEA1YPgvqcu +yOkkI3udtSsM3jvN//u6U9wSC1eE1KoWc8x4ia9Jr1B1Y4mdO+IaIdg6v2x5AzIt +aj5PrnMo+y3nvbndmEBmNPehCbzxUMz+GfNW6zf13Av1sywjALQiwQ2p6vFIFoIX +vMcesd4ZO53CPPisEGOY3qXLzgbEDhyM/zrTPbgNusYvvkzY67PtcGzVlSlwPpJV ++4Hk+ztWrCxNqtl3hjUObBjnuKjGxt+hN2iDfgoVG97yhPzkB8C6AbNTaZqvqg2C +naUx5MOxcll661s7V5bTo4XwEuWWXKj5c2wkO60AHvkjYc7me3i6jlg9YlyPgOi+ +6f0V2u3VkX7l2wKCAQEAovFkigRYFJPSbt1qV2txINIbQSggYzCFhQoJ9wBdiCrw +9KlV+tsmp/uSzEIsz43OwQ/BIwnpUQM9T4K0mLVrNcIUtwiEisjMMk3SLlEtqP3U +aAffm3Jj68WG0vVYRpSa1Y5rvitvygH7v0RbTYygMYTD7FFKWmH+nRlFZrcUs73e +RGuV817Gzc2j06NledxJNMEdVoGcWOtlsYCobs1/yZvK/gujEHL2nbj34XwTy8rk +OdFvJlux3z05sFXyn8K6PnPZldeTnXJCi9Fqxc4z2BCJDQm6wSzpZZUnU1RRhbc8 +b3b3HEia4rf/QWS/8O7Gxo7PS1dhp12f2s1peYk9PwKCAQAxFYIjEhflRAN0zMQy +k9T/ecwfnuT0xlC3nsUAhqFmuYi0TkGoNdzmpwoobBAJ28WVoAApxe0+0VhMCFlR +dPojWYkhqRxV7N9ud6saIiYAHTrMFC9HCNDRAcKCNOcQbm2zfwhNdFa0pSnfRemT +FO9ESP51PhA0jvTNRizn+ZRIUGOjep5dY5YyL0Rm2xQoljx7b+1H1ShDC1dyke+Y +4Q5xylB539SS8R7ECri3m01aiYJBBVxY7eXewKxDRAD+xxTT4CWl+DkguItBxeMT +IJLrbCu2NQwuOWo5TeJFJutBp4ik116BwFBr+b5ugBCTDKH/7LtorRjGfdH6ZFaG +fh+lAoIBABKafHZT9YZPUkvHB5Zs2V5G43PvXv/FM48QBzWGtqLRNnKKtI/JgENV +M1MflSKQLqCRCw/ctHzfPI6zVR7a5AagellJeX/OPJdHN8OiIGHoXtkSHkWETNG3 +JuIN+MgIC/Y7vXtRUt6RwQpkBYq0OlZ4X+2mOYNl4ORglA6OhJnTPzYYxdTeWs/c +00AgkFi8YmITTaAHAG5f609zTz8/LbDie3jwvy0ORiHKfL+B+ihR27zRmNZ/rSbv +9M9m9bKViZoC6Zf/7hN2l7Pjl9IgkUx9Oy3kJSLYqaFpSicN7fGaSkQuw3VDgKvB +Tc4qvMOITdZUl1Das9MqYWp4G8Uk8CA= +-----END PRIVATE KEY----- diff --git a/test/net/integration/microLB/drive/test.pem b/test/net/integration/microLB/drive/test.pem new file mode 100644 index 0000000000..e6ddc43a1a --- /dev/null +++ b/test/net/integration/microLB/drive/test.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFXTCCA0WgAwIBAgIJAMKoGEJAbpuNMA0GCSqGSIb3DQEBDQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTcwMzA5MTM0NTI2WhcNMjcwMzA3MTM0NTI2WjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAszXcDz3rlOfT4mgkqpJmhWEA1oSmXJl+ODWgAM5ozVLzXvbvAxJovjG5 +mRmDJOb0zRpD6xjOcWVSrOLqyr7YKFhDK2N7hWMmKKjTaWl640uWogWMdomQ1nQ2 +wdLKyzoepYhY35GVWzDb2oLwG/zVfC+mwVrQOeDZgOQtFanwd9HURipuwld8IC8h +4JgcB2i99VZGoQXExhRQlcNUGUzUaExDoQMG/+K7e2DmZDzjsmsAnOgeDVGd2CeA +u8WydLenL+HP04qH6bVgxETVdhQ5GXYfSS1ZyRkLavCM/QftLYYCo/ibJ3mK9OCr +EPiKfBM6as3qgTRVp83IqPIgURcVJPRLEAW0i8SpsXvqA56M558NRJ06I4PIT7xH +A7TvR6wD8GH2u7W4+c5AaQwTxX2C4Gbtg/wESEwlSzgfruHGaNIyI4nF97m1hF7x +VbcHXBpvjFK2+6gzBTNJ8isNJL4XFaGR1x1Gm03NYQhkFVGiOaOGF6VEL9uXIbSz +XaHBDePz7nvPZXYytYo93H8AdM/nkoQ5PTOx/6a/SOwgtSjMZAG6TohGI/1H6zvm +5ToF33O7B+nDkd5bSO/YNsVuDN5NYDwNuBw2i7dVKTWb4E1IcvTAI3rKY1DDd1y9 +VpyYZSQvsnWatjKiVwANnbYNitNM0zSvPDufKzvSGq4JraHlK0kCAwEAAaNQME4w +HQYDVR0OBBYEFAHh+il41QCAviLPiUnybxnw8vDCMB8GA1UdIwQYMBaAFAHh+il4 +1QCAviLPiUnybxnw8vDCMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQENBQADggIB +AIG9NyK5uD5A11VI6gHC8cmfnVszHuaLpuc2y7jyVCyZOt38a0YmKGTUD/u+JKUc +C01uREEi/tqJsm1yE9/kxMMlCGSW9/2kKbbkgU9LdVyVQxr4yVA2aHOf6GDfEioT +O1WXPITZyjBUF5N/t6+LCyClF+0CsDnzWkZkIXEiY5W4Nlqctq1Ef4wMzvCVQ5t6 +RUX8UpxZuraENjtiZlFP8HjQER7/QK+eQl+tI56snAnyU2AcABcUegP1/IphMCDL +iAAGkqBu1wndeeIwUtT9LHuVGWVGEsr1Zpq0NJuVmU5E0i1VRl7whBJQw+PMqFde +6CwI/pMyuZEgh+tXVx8VakI4eSJX8TRPaLeMSI+Sr3CfZxm7Zu6YUEXSLgMrwRqY +atx0oGL8lnYTxlozG+Zmdytge8OwwgtBgBHlmiqbko3ePvlSZTw3gx7tpFBByzX9 +TbgiVKA9P+j5lhdxScls8WjUkr2wXJBXv9WPIK0pUP4UzLJQltamo9lOUJiPgzf9 +v8yPfI4clt4YpRAZw/h1wjhHSYYLmtF6Y8K50NgiFAMuXq5G7q9SlAISt+9oGojA +80MLVkzin8zNuAvGGKnJ72vggKIqIqpdNUyWFH4yuhOtorN+Xam8uPb3axKjVkeP +x2tytEIsJgJY/xggNU1h8myeo8EJLwgXeEVQWMOiE2jI +-----END CERTIFICATE----- diff --git a/test/net/integration/microLB/test.py b/test/net/integration/microLB/test.py index c410df625c..68487f8a44 100755 --- a/test/net/integration/microLB/test.py +++ b/test/net/integration/microLB/test.py @@ -14,8 +14,8 @@ import requests def validateRequest(expected = ""): - response = requests.get('http://10.0.0.68') - #print (response.content) + response = requests.get('https://10.0.0.68:443', verify=False) + print (response.content) return (response.content) == expected # start nodeJS From 0ad74d2a4e513d412f9a0d5139f34fa442e1ebc0 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 19 Apr 2018 13:40:21 +0200 Subject: [PATCH 202/723] openssl: Clear errors before calling OpenSSL in TLS stream --- api/net/openssl/tls_stream.hpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index c745e20df6..5f5a4bf3bc 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -97,6 +97,7 @@ namespace openssl inline TLS_stream::TLS_stream(SSL_CTX* ctx, Stream_ptr t, bool outgoing) : m_transport(std::move(t)) { + ERR_clear_error(); // prevent old errors from mucking things up this->m_bio_rd = BIO_new(BIO_s_mem()); this->m_bio_wr = BIO_new(BIO_s_mem()); assert(ERR_get_error() == 0 && "Initializing BIOs"); @@ -155,6 +156,7 @@ namespace openssl inline void TLS_stream::tls_read(buffer_t buffer) { + ERR_clear_error(); uint8_t* buf = buffer->data(); int len = buffer->size(); @@ -229,6 +231,7 @@ namespace openssl inline int TLS_stream::tls_perform_stream_write() { + ERR_clear_error(); int pending = BIO_ctrl_pending(this->m_bio_wr); //printf("pending: %d\n", pending); if (pending > 0) @@ -252,6 +255,7 @@ namespace openssl } inline int TLS_stream::tls_perform_handshake() { + ERR_clear_error(); // prevent old errors from mucking things up // will return -1:SSL_ERROR_WANT_WRITE int ret = SSL_do_handshake(this->m_ssl); int n = this->status(ret); @@ -275,6 +279,7 @@ namespace openssl inline void TLS_stream::close() { + ERR_clear_error(); m_transport->close(); } inline void TLS_stream::abort() From a63f1d4ef1c4ec8a2193d2c2f64b702b887e08e2 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 19 Apr 2018 14:17:04 +0200 Subject: [PATCH 203/723] heap: remove OS::resize_heap, update memory_map test --- api/kernel/os.hpp | 3 - src/kernel/heap.cpp | 10 -- test/kernel/integration/memmap/service.cpp | 105 ++++++++------------- 3 files changed, 40 insertions(+), 78 deletions(-) diff --git a/api/kernel/os.hpp b/api/kernel/os.hpp index fec5c7efdb..7845708f40 100644 --- a/api/kernel/os.hpp +++ b/api/kernel/os.hpp @@ -189,9 +189,6 @@ class OS { /** Last used address of the heap **/ static uintptr_t heap_end() noexcept; - /** Resize the heap if possible. Return (potentially) new size. **/ - static uintptr_t resize_heap(size_t size); - /** The maximum last address of the dynamic memory area (heap) */ static uintptr_t heap_max() noexcept; diff --git a/src/kernel/heap.cpp b/src/kernel/heap.cpp index 5db56065ec..422bf9cf72 100644 --- a/src/kernel/heap.cpp +++ b/src/kernel/heap.cpp @@ -60,16 +60,6 @@ uintptr_t OS::heap_end() noexcept return mmap_allocation_end(); } -uintptr_t OS::resize_heap(size_t size) -{ - uintptr_t new_end = heap_begin() + size; - if (not size or size < heap_usage() or new_end > memory_end()) - return heap_max() - heap_begin(); - - memory_map().resize(heap_begin(), size); - heap_max_ = heap_begin() + size; - return size; -} constexpr size_t heap_alignment = 4096; __attribute__((weak)) ssize_t __brk_max = 0x100000; diff --git a/test/kernel/integration/memmap/service.cpp b/test/kernel/integration/memmap/service.cpp index 2262270889..3b3fe2e902 100644 --- a/test/kernel/integration/memmap/service.cpp +++ b/test/kernel/integration/memmap/service.cpp @@ -16,80 +16,55 @@ // limitations under the License. #include -#include +#include #include -extern "C" uintptr_t heap_begin; -extern "C" uintptr_t heap_end; using namespace util; -const lest::test specification[] = - { - { - CASE( "Make sure the heap respects the memory map max" ) - { - auto heap = OS::memory_map().at(heap_begin); - EXPECT(heap.addr_start()); - EXPECT(heap.addr_end() == OS::heap_max()); - auto* buf = malloc(0x100000); - EXPECT(buf); - free(buf); - EXPECT_NOT(malloc(heap.addr_end())); - EXPECT(errno == ENOMEM); - } - }, - { - SCENARIO ("Using the kernel memory map") - { - GIVEN ("The kernel map") - { - auto& map = OS::memory_map(); - EXPECT(map.size()); - - THEN("You can resize the heap as long as it's not full") - { - auto& heap = map.at(heap_begin); - auto original_end = heap.addr_end(); - EXPECT(original_end == OS::heap_max()); - auto in_use_prior = bits::roundto(4_KiB, heap.bytes_in_use()); - - EXPECT(heap.bytes_in_use() < heap.size()); - - // Resize heap to have only 1 MB of free space - auto new_size = in_use_prior + 1_MiB; - OS::resize_heap(new_size); - - EXPECT(heap.addr_end() == heap.addr_start() + in_use_prior + 1_MiB - 1); - EXPECT(heap.addr_end() == OS::heap_max() - 1); - - ptrdiff_t size = 1000; - map.assign_range({OS::heap_max(), OS::heap_max() + size - 1, "Custom"}); - - for (auto i: map) - std::cout << i.second.to_string() << "\n"; - - auto capacity = heap.addr_end() - heap.addr_start() + heap.bytes_in_use(); - EXPECT(capacity > 1_MiB); - - EXPECT_NOT(malloc(2_MiB)); - - // Malloc might ask sbrk for more than it needs - auto* buf = malloc(500_KiB); - EXPECT(buf); - } - } - } - } - }; - - void Service::start(const std::string&) { INFO("Memmap", "Testing the kernel memory map"); - auto failed = lest::run(specification, {"-p"}); - Expects(not failed); + auto& map = OS::memory_map(); + Expects(map.size()); + + // Verify that you can't create any overlapping ranges + const auto s = map.size(); + int failed = 0; + auto i = 0; + for (auto it : map) { + try { + auto m = it.second; + Expects(m.size()); + int offs = rand() % m.size() + 1; + uintptr_t begin = i++ % 2 ? m.addr_start() + offs : m.addr_start() - offs; + uintptr_t end = begin + offs * 2; + Fixed_memory_range rng {begin, end, "Can't work"}; + map.assign_range(std::move(rng)); + } catch (const std::exception& e) { + failed++; + } + } + + Expects(failed == s); + Expects(map.size() == s); + + // mem::map is using memory_map to keep track of virutal memory + // TODO: we might consider consolidating ranges with mappings. + auto m = os::mem::map({42_GiB, 42_GiB, os::mem::Access::read, 1_GiB}, + "Test range"); + Expects(m); + Expects(map.size() == s + 1); + + auto key = map.in_range(42_GiB); + Expects(key == 42_GiB); + auto rng = map.at(key); + Expects(rng.size() == 1_GiB); + + // TODO: There's an unimplemented map.assign_range(size) to integrate. + // auto& given = map.assign_range(5_GiB); + // Expects(given.size() == 5_GiB); INFO("Memmap", "SUCCESS"); From 4491c735c00a56d5541c6f3c82d8c3c163ec2221 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 19 Apr 2018 14:33:24 +0200 Subject: [PATCH 204/723] microLB: Add dependency on OpenSSL libraries, build only on 64-bit --- lib/microLB/CMakeLists.txt | 46 ++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/lib/microLB/CMakeLists.txt b/lib/microLB/CMakeLists.txt index 007d3ffebc..622b5a4de8 100644 --- a/lib/microLB/CMakeLists.txt +++ b/lib/microLB/CMakeLists.txt @@ -1,28 +1,30 @@ cmake_minimum_required(VERSION 2.8.9) -add_definitions(-DARCH_${ARCH}) -add_definitions(-DARCH="${ARCH}") +if (${ARCH} STREQUAL "x86_64") + add_definitions(-DARCH_${ARCH}) + add_definitions(-DARCH="${ARCH}") -# microLB static library -add_library(microlb STATIC - micro_lb/autoconf.cpp - micro_lb/balancer.cpp - micro_lb/openssl.cpp - micro_lb/serialize.cpp - ) + # microLB static library + add_library(microlb STATIC + micro_lb/autoconf.cpp + micro_lb/balancer.cpp + micro_lb/openssl.cpp + micro_lb/serialize.cpp + ) -add_dependencies(microlb PrecompiledLibraries) + add_dependencies(microlb PrecompiledLibraries openssl_bundle) -target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/api/posix) -target_include_directories(microlb PUBLIC ${LIBCXX_INCLUDE_DIR}) -target_include_directories(microlb PUBLIC ${MUSL_INCLUDE_DIR}) -target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/src/include) -target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/api) -target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/lib/LiveUpdate) -target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/mod/rapidjson/include) -target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/mod/GSL/) -target_include_directories(microlb PUBLIC ${OPENSSL_DIR}/include) + target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/api/posix) + target_include_directories(microlb PUBLIC ${LIBCXX_INCLUDE_DIR}) + target_include_directories(microlb PUBLIC ${MUSL_INCLUDE_DIR}) + target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/src/include) + target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/api) + target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/lib/LiveUpdate) + target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/mod/rapidjson/include) + target_include_directories(microlb PUBLIC ${INCLUDEOS_ROOT}/mod/GSL/) + target_include_directories(microlb PUBLIC ${OPENSSL_DIR}/include) -install(TARGETS microlb DESTINATION includeos/${ARCH}/lib) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/micro_lb/balancer.hpp DESTINATION includeos/include/micro_lb) -install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/microLB DESTINATION includeos/include) + install(TARGETS microlb DESTINATION includeos/${ARCH}/lib) + install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/micro_lb/balancer.hpp DESTINATION includeos/include/micro_lb) + install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/microLB DESTINATION includeos/include) +endif() From 43e17788e8e280dcce1d391680f4486093d7ebe0 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 19 Apr 2018 15:44:55 +0200 Subject: [PATCH 205/723] heap: set memory_end default to same as heap_max --- src/kernel/heap.cpp | 2 +- src/platform/x86_pc/kernel_start.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/kernel/heap.cpp b/src/kernel/heap.cpp index 422bf9cf72..df4924f566 100644 --- a/src/kernel/heap.cpp +++ b/src/kernel/heap.cpp @@ -28,7 +28,7 @@ static constexpr size_t default_max_mem = 2_GiB; uintptr_t OS::heap_begin_ = 0; uintptr_t OS::heap_max_ = default_max_mem; -uintptr_t OS::memory_end_ = __arch_max_canonical_addr; +uintptr_t OS::memory_end_ = default_max_mem; size_t OS::heap_usage() noexcept diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index e0b66275e7..c69d94c02c 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -131,9 +131,9 @@ void kernel_start(uint32_t magic, uint32_t addr) if (magic == MULTIBOOT_BOOTLOADER_MAGIC) { free_mem_begin = _multiboot_free_begin(addr); memory_end = _multiboot_memory_end(addr); - kprintf("* Free mem begin: 0x%zx, memory end: 0x%zx \n", - free_mem_begin, memory_end); } + kprintf("* Free mem begin: 0x%zx, memory end: 0x%zx \n", + free_mem_begin, memory_end); kprintf("* Moving symbols. \n"); // Preserve symbols from the ELF binary From 6d63ff90ae3df453f94b9a27f362130b0d3ef284 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Fri, 20 Apr 2018 08:55:31 +0200 Subject: [PATCH 206/723] install.sh: Optional -s flag to skip installing dependencies --- install.sh | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/install.sh b/install.sh index 7b698a3919..2d66be7ee5 100755 --- a/install.sh +++ b/install.sh @@ -24,15 +24,17 @@ install_yes=0 quiet=0 bundle_location="" net_bridge=1 +skip_dependencies=0 -while getopts "h?yqb:n" opt; do +while getopts "h?yqb:ns" opt; do case "$opt" in h|\?) printf "%s\n" "Options:"\ "-y Yes: answer yes to install"\ "-q Quiet: Suppress output from cmake during install"\ "-b Bundle: Local path to bundle"\ - "-n No Net bridge: Disable setting up network bridge" + "-n No Net bridge: Disable setting up network bridge"\ + "-s Skip dependencies: Don't check for dependencies" exit 0 ;; y) install_yes=1 @@ -49,6 +51,8 @@ while getopts "h?yqb:n" opt; do ;; n) net_bridge=0 ;; + s) skip_dependencies=1 + ;; esac done @@ -114,25 +118,24 @@ if ! command -v sudo > /dev/null 2>&1; then fi # Install build requirements (compiler, etc) -if [ "Darwin" = "$SYSTEM" ]; then - echo ">>> Dependencies required:" - if ! ./etc/install_dependencies_macos.sh -c; then - missing_dependencies=1 - fi -else - # Will only check if build dependencies are installed at this point - if [ $INCLUDEOS_ENABLE_TEST == "ON" ]; then - dependency_level=all - else - dependency_level=build - fi - if [ $do_packages ] - then - echo ">>> Dependencies required:" - if ! ./etc/install_dependencies_linux.sh -s $SYSTEM -r $RELEASE -c -d $dependency_level; then - missing_dependencies=1 +if [ $skip_dependencies -eq 0 ]; then + if [ "Darwin" = "$SYSTEM" ]; then + echo ">>> Dependencies required:" + if ! ./etc/install_dependencies_macos.sh -c; then + missing_dependencies=1 + fi + else + # Will only check if build dependencies are installed at this point + if [ $INCLUDEOS_ENABLE_TEST == "ON" ]; then + dependency_level=all + else + dependency_level=build + fi + echo ">>> Dependencies required:" + if ! ./etc/install_dependencies_linux.sh -s $SYSTEM -r $RELEASE -c -d $dependency_level; then + missing_dependencies=1 fi - fi + fi fi ############################################################ From 1c4ebc821d3a9ac3abf392008e2c8d2850aac433 Mon Sep 17 00:00:00 2001 From: Per Buer Date: Fri, 20 Apr 2018 09:16:39 +0200 Subject: [PATCH 207/723] remove compiler warning (unused variable) --- src/plugins/field_medic/fieldmedic.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/plugins/field_medic/fieldmedic.cpp b/src/plugins/field_medic/fieldmedic.cpp index de37c69e4b..99ddd7b99f 100644 --- a/src/plugins/field_medic/fieldmedic.cpp +++ b/src/plugins/field_medic/fieldmedic.cpp @@ -70,7 +70,6 @@ void init(){ if(c != '!') throw medic::diag::Error("unexpected .tbss value"); } - int i = 0; for (auto& i : __tl_data) { if (i != 42) return false; From 38d12182d82237691796fb53398d828cfe2e8df8 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 20 Apr 2018 09:46:32 +0200 Subject: [PATCH 208/723] Revert LiveUpdate: clear memory on amd64 during update --- lib/LiveUpdate/hotswap64.asm | 16 +--------------- lib/LiveUpdate/update.cpp | 5 ++--- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/lib/LiveUpdate/hotswap64.asm b/lib/LiveUpdate/hotswap64.asm index 5c053d0332..1f001a4f6c 100644 --- a/lib/LiveUpdate/hotswap64.asm +++ b/lib/LiveUpdate/hotswap64.asm @@ -33,8 +33,7 @@ ALIGN 16 ;; RSI: const char* base, ;; RDX: size_t len, ;; RCX: void* entry_function, -;; R8: void* reset_data, -;; R9: void* liu_storage) +;; R8: void* reset_data) hotswap_amd64: ;; save soft reset data location and entry function mov rax, r8 @@ -48,19 +47,6 @@ hotswap_amd64: cld rep movsb - ;; clear heap + kernel remains - ;; RDI already at end of new kernel - mov rax, 0 - mov rcx, r9 ;; LU location - sub rcx, rdi ;; - end of new kernel - rep stosb - - ;; clear stack - mov rdi, 0x100000 - mov rax, 0 - mov rcx, 0x100000 - rep stosb - begin_enter_protected: ; load 64-bit GDTR with 32-bit entries lgdt [gdtr64] diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index afe3155693..c2de1c7d00 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -41,7 +41,7 @@ extern "C" void solo5_exec(const char*, size_t); static void* HOTSWAP_AREA = (void*) 0x8000; extern "C" void hotswap(const char*, int, char*, uintptr_t, void*); extern "C" char __hotswap_length; -extern "C" void hotswap64(char*, const char*, int, uintptr_t, void*, void*); +extern "C" void hotswap64(char*, const char*, int, uintptr_t, void*); extern uint32_t hotswap64_len; extern void __x86_init_paging(void*); extern "C" void* __os_store_soft_reset(const void*, size_t); @@ -236,8 +236,7 @@ void LiveUpdate::exec(const buffer_t& blob) // copy hotswapping function to sweet spot memcpy(HOTSWAP_AREA, (void*) &hotswap64, hotswap64_len); /// the end - char* end_loc = storage_area - 0x110000; - ((decltype(&hotswap64)) HOTSWAP_AREA)(phys_base, bin_data, bin_len, start_offset, sr_data, end_loc); + ((decltype(&hotswap64)) HOTSWAP_AREA)(phys_base, bin_data, bin_len, start_offset, sr_data); # else # error "Unimplemented architecture" # endif From b50e55b954a148e1ba286991f00dc3be5875bbcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 20 Apr 2018 10:21:23 +0200 Subject: [PATCH 209/723] uplink: Support https, only works on 64bit due to openssl --- lib/uplink/CMakeLists.txt | 5 ++++ lib/uplink/config.cpp | 5 ++++ lib/uplink/config.hpp | 3 +++ lib/uplink/ws_uplink.cpp | 48 ++++++++++++++++++++++++++++++++++++-- lib/uplink/ws_uplink.hpp | 2 +- src/net/openssl/client.cpp | 2 +- 6 files changed, 61 insertions(+), 4 deletions(-) diff --git a/lib/uplink/CMakeLists.txt b/lib/uplink/CMakeLists.txt index 1fea6c3a1e..46b3f6373f 100644 --- a/lib/uplink/CMakeLists.txt +++ b/lib/uplink/CMakeLists.txt @@ -1,5 +1,6 @@ cmake_minimum_required(VERSION 2.8.9) +if (${ARCH} STREQUAL "x86_64") add_definitions(-DARCH_${ARCH}) add_definitions(-DARCH="${ARCH}") @@ -13,6 +14,7 @@ include_directories(${INCLUDEOS_ROOT}/mod/GSL/) #dependencies include_directories(${INCLUDEOS_ROOT}/lib/LiveUpdate) include_directories(${INCLUDEOS_ROOT}/mod/rapidjson/include) +include_directories(${OPENSSL_DIR}/include) set(LIBRARY_NAME "uplink") @@ -26,9 +28,12 @@ set(SOURCES # Uplink library add_library(${LIBRARY_NAME} STATIC ${SOURCES}) add_dependencies(${LIBRARY_NAME} PrecompiledLibraries) +add_dependencies(${LIBRARY_NAME} openssl_bundle) install(TARGETS ${LIBRARY_NAME} DESTINATION includeos/${ARCH}/plugins) # Uplink log driver add_library(uplink_log STATIC uplink_log.cpp) add_dependencies(uplink_log PrecompiledLibraries) +add_dependencies(uplink_log openssl_bundle) install(TARGETS uplink_log DESTINATION includeos/${ARCH}/drivers) +endif() diff --git a/lib/uplink/config.cpp b/lib/uplink/config.cpp index 33f609cc56..77026e8230 100644 --- a/lib/uplink/config.cpp +++ b/lib/uplink/config.cpp @@ -101,6 +101,11 @@ namespace uplink { config_.tag = cfg["tag"].GetString(); } + if(cfg.HasMember("certs")) + { + config_.certs_path = cfg["certs"].GetString(); + } + return config_; } diff --git a/lib/uplink/config.hpp b/lib/uplink/config.hpp index 3879154861..ea6d5c0084 100644 --- a/lib/uplink/config.hpp +++ b/lib/uplink/config.hpp @@ -25,12 +25,15 @@ namespace uplink { + const static std::string default_cert_path{"/certs"}; + struct Config { net::Inet* inet; std::string url; std::string token; std::string tag; + std::string certs_path = default_cert_path; bool reboot = true; bool ws_logging = true; bool serialize_ct = false; diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index a7b52bb998..e4d0593909 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -40,6 +40,35 @@ #include #include +#include +#include + +static SSL_CTX* init_ssl_context(const std::string& certs_path) +{ + MYINFO("Reading certificates from disk @ %s", certs_path.c_str()); + auto& disk = fs::memdisk(); + disk.init_fs([] (auto err, auto&) { + Ensures(!err && "Error init filesystem"); + }); + + auto ents = disk.fs().ls(certs_path); + + int files = 0; + MYINFO("Scanning files..."); + for(auto& ent : ents) { + if(not ent.is_file()) + continue; + INFO2("%s", ent.name().c_str()); + files++; + } + + Expects(files > 0 && "No files found on disk"); + + // initialize client context + openssl::init(); + return openssl::create_client(ents, true); +} + namespace uplink { constexpr std::chrono::seconds WS_uplink::heartbeat_interval; @@ -90,8 +119,23 @@ namespace uplink { Expects(inet.ip_addr() != 0 && "Network interface not configured"); Expects(not config_.url.empty()); - client_ = std::make_unique(inet.tcp(), - http::Basic_client::Request_handler{this, &WS_uplink::inject_token}); + const uri::URI url{config_.url}; + + Expects(url.is_valid() && "Invalid URL"); + + if(url.scheme() == "https") + { + auto* ssl_context = init_ssl_context(config_.certs_path); + Expects(ssl_context != nullptr && "Secure URL given but no valid certificates found"); + + client_ = std::make_unique(inet.tcp(), ssl_context, + http::Basic_client::Request_handler{this, &WS_uplink::inject_token}); + } + else + { + client_ = std::make_unique(inet.tcp(), + http::Basic_client::Request_handler{this, &WS_uplink::inject_token}); + } auth(); } diff --git a/lib/uplink/ws_uplink.hpp b/lib/uplink/ws_uplink.hpp index 2f1518290f..e7a21f3b6e 100644 --- a/lib/uplink/ws_uplink.hpp +++ b/lib/uplink/ws_uplink.hpp @@ -23,7 +23,7 @@ #include "config.hpp" #include -#include +#include #include #include #include diff --git a/src/net/openssl/client.cpp b/src/net/openssl/client.cpp index dafbf584bc..30e15cff8b 100644 --- a/src/net/openssl/client.cpp +++ b/src/net/openssl/client.cpp @@ -28,7 +28,7 @@ tls_load_from_memory(X509_STORE* store, { auto* cbio = BIO_new_mem_buf(cert_buffer.data(), cert_buffer.size()); auto* cert = PEM_read_bio_X509(cbio, NULL, 0, NULL); - assert(cert != NULL); + assert(cert != NULL && "Invalid certificate"); int res = X509_STORE_add_cert(store, cert); assert(res == 1); BIO_free(cbio); From 0bc88807a328a88f65ec0e064f50dd0c157a2f31 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 20 Apr 2018 10:35:18 +0200 Subject: [PATCH 210/723] cmake: Install CA-bundle in openssl.cmake --- cmake/openssl.cmake | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/cmake/openssl.cmake b/cmake/openssl.cmake index 925a81da84..4dab262690 100644 --- a/cmake/openssl.cmake +++ b/cmake/openssl.cmake @@ -28,3 +28,15 @@ if(${ARCH} STREQUAL "x86_64") install(FILES ${OPENSSL_LIB_SSL} DESTINATION includeos/${ARCH}/lib) install(DIRECTORY ${OPENSSL_INCLUDE} DESTINATION includeos/${ARCH}) endif() + +ExternalProject_Add(cert_bundle + PREFIX cert_bundle + URL https://github.com/fwsGonzo/OpenSSL_bundle/releases/download/v1.2/ca_bundle.tar.gz + URL_HASH MD5=4596f90b912bea7ad7bd974d10c58efd + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + UPDATE_COMMAND "" + INSTALL_COMMAND "" +) +set(CERT_BUNDLE_DIR ${CMAKE_CURRENT_BINARY_DIR}/cert_bundle/src/cert_bundle) +install(DIRECTORY ${CERT_BUNDLE_DIR} DESTINATION includeos/) From 3a3fc3a92a1ae096b14e33982efd894cb5df7eb6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 20 Apr 2018 11:24:13 +0200 Subject: [PATCH 211/723] cmake: Add install_certificates function for OpenSSL --- cmake/post.service.cmake | 5 +++++ examples/http_client/.gitignore | 1 + examples/http_client/CMakeLists.txt | 5 ++++- examples/http_client/config.json | 5 ----- examples/http_client/service.cpp | 2 +- 5 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 examples/http_client/.gitignore delete mode 100644 examples/http_client/config.json diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index f5ab771c56..d36967cd5b 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -399,6 +399,11 @@ function(diskbuilder FOLD) endif() endfunction() +function(install_certificates FOLD) + get_filename_component(REL_PATH "${FOLD}" REALPATH BASE_DIR "${CMAKE_SOURCE_DIR}") + file(COPY ${INSTALL_LOC}/cert_bundle DESTINATION ${REL_PATH}) +endfunction() + if(TARFILE) get_filename_component(TAR_RELPATH "${TARFILE}" REALPATH BASE_DIR "${CMAKE_SOURCE_DIR}") diff --git a/examples/http_client/.gitignore b/examples/http_client/.gitignore new file mode 100644 index 0000000000..d6e02fb0c7 --- /dev/null +++ b/examples/http_client/.gitignore @@ -0,0 +1 @@ +disk/ diff --git a/examples/http_client/CMakeLists.txt b/examples/http_client/CMakeLists.txt index 8ec97480a6..975d47d151 100644 --- a/examples/http_client/CMakeLists.txt +++ b/examples/http_client/CMakeLists.txt @@ -26,8 +26,11 @@ set(DRIVERS #boot_logger ) -set(PLUGINS vfs) +set(PLUGINS + #vfs + ) include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) +install_certificates(disk) diskbuilder(disk) diff --git a/examples/http_client/config.json b/examples/http_client/config.json deleted file mode 100644 index c42702a586..0000000000 --- a/examples/http_client/config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "vfs": [ - {"disk": "memdisk", "root": "/mozilla", "mount": "/certs", "description": ""} - ] -} diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index afa0ea5c3f..9f7c4d8aa9 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -27,7 +27,7 @@ static SSL_CTX* init_ssl_context() }); acorn::list_static_content(disk.fs()); - auto ents = disk.fs().ls("/mozilla"); + auto ents = disk.fs().ls("/cert_bundle"); // initialize client context openssl::init(); From 56b7b233738fd5a4141e67a25d62e78d9007b857 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 20 Apr 2018 11:44:56 +0200 Subject: [PATCH 212/723] examples: Remove acorn dependency in http_client --- examples/http_client/CMakeLists.txt | 1 - examples/http_client/service.cpp | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/examples/http_client/CMakeLists.txt b/examples/http_client/CMakeLists.txt index 975d47d151..34d1726b03 100644 --- a/examples/http_client/CMakeLists.txt +++ b/examples/http_client/CMakeLists.txt @@ -16,7 +16,6 @@ set(BINARY "http_client") # Source files to be linked with OS library parts to form bootable image set(SOURCES service.cpp - ../acorn/fs/acorn_fs.cpp ) set(DRIVERS diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index 9f7c4d8aa9..f07a19ee6a 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -17,18 +17,18 @@ #include #include -#include "../acorn/fs/acorn_fs.hpp" static SSL_CTX* init_ssl_context() { auto& disk = fs::memdisk(); - disk.init_fs([] (auto err, auto&) { + disk.init_fs([] (fs::error_t err, auto& fs) { assert(!err); + + err = fs.print_subtree("/cert_bundle"); + assert(err == fs::no_error && "Need certificate bundle folder present"); }); - acorn::list_static_content(disk.fs()); auto ents = disk.fs().ls("/cert_bundle"); - // initialize client context openssl::init(); return openssl::create_client(ents, true); From 5f96de794f350f7dfa546af5add6b2c0f871d237 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 20 Apr 2018 15:14:35 +0200 Subject: [PATCH 213/723] util: Added secure-helper for URI, always parse when appending --- api/util/uri.hpp | 7 +++++++ src/net/http/basic_client.cpp | 4 ++-- src/util/uri.cpp | 6 ++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/api/util/uri.hpp b/api/util/uri.hpp index 1956cc99de..daebb85db8 100644 --- a/api/util/uri.hpp +++ b/api/util/uri.hpp @@ -136,6 +136,13 @@ class URI { /// util::sview scheme() const noexcept; + /** + * @brief Check whether the scheme is secure (like https or wss) or not. + * + * @return true if secure, false otherwise + */ + bool scheme_is_secure() const noexcept; + /// /// Get userinfo. /// diff --git a/src/net/http/basic_client.cpp b/src/net/http/basic_client.cpp index b6caa39852..0131a32d0e 100644 --- a/src/net/http/basic_client.cpp +++ b/src/net/http/basic_client.cpp @@ -77,7 +77,7 @@ namespace http { Expects(cb != nullptr); // find out if this is a secured request or not - const bool secure = (url.scheme() == "https"); + const bool secure = url.scheme_is_secure(); validate_secure(secure); using namespace std; @@ -158,7 +158,7 @@ namespace http { Options options) { // find out if this is a secured request or not - const bool secure = (url.scheme() == "https"); + const bool secure = url.scheme_is_secure(); validate_secure(secure); using namespace std; diff --git a/src/util/uri.cpp b/src/util/uri.cpp index 53f0efbbec..5d4ce3d63c 100644 --- a/src/util/uri.cpp +++ b/src/util/uri.cpp @@ -58,6 +58,7 @@ static inline uint16_t bind_port(util::csview scheme, const uint16_t port_from_u {"ssh", 22U}, {"telnet", 23U}, {"ws", 80U}, + {"wss", 443U}, {"xmpp", 5222U}, }; @@ -186,6 +187,10 @@ util::sview URI::scheme() const noexcept { return scheme_; } +bool URI::scheme_is_secure() const noexcept { + return scheme() == "https" or scheme() == "wss"; +} + /////////////////////////////////////////////////////////////////////////////// util::sview URI::userinfo() const noexcept { return userinfo_; @@ -270,6 +275,7 @@ URI::operator std::string () const { /////////////////////////////////////////////////////////////////////////////// URI& URI::operator << (const std::string& chunk) { uri_str_.insert(uri_str_.end(), chunk.begin(), chunk.end()); + parse(); return *this; } From 03fce7913047d4d86a3bec980152b8f9e6c3e24e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 20 Apr 2018 15:18:03 +0200 Subject: [PATCH 214/723] build/examples: Can now decide dir name for certificates, HTTP example fixed #1745 --- api/net/openssl/tls_stream.hpp | 2 +- cmake/post.service.cmake | 3 +- examples/http_client/CMakeLists.txt | 2 +- examples/http_client/disk/server.pem | 83 ---------------------------- examples/http_client/service.cpp | 8 +-- 5 files changed, 8 insertions(+), 90 deletions(-) delete mode 100644 examples/http_client/disk/server.pem diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index 5f5a4bf3bc..cee495dfed 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -4,7 +4,7 @@ #include #include -#define VERBOSE_OPENSSL +//#define VERBOSE_OPENSSL #ifdef VERBOSE_OPENSSL #define TLS_PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index d36967cd5b..f86bfc1fdf 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -401,7 +401,8 @@ endfunction() function(install_certificates FOLD) get_filename_component(REL_PATH "${FOLD}" REALPATH BASE_DIR "${CMAKE_SOURCE_DIR}") - file(COPY ${INSTALL_LOC}/cert_bundle DESTINATION ${REL_PATH}) + message(STATUS "Install certificate bundle at ${FOLD}") + file(COPY ${INSTALL_LOC}/cert_bundle/ DESTINATION ${REL_PATH}) endfunction() if(TARFILE) diff --git a/examples/http_client/CMakeLists.txt b/examples/http_client/CMakeLists.txt index 34d1726b03..1e9f225ba9 100644 --- a/examples/http_client/CMakeLists.txt +++ b/examples/http_client/CMakeLists.txt @@ -31,5 +31,5 @@ set(PLUGINS include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) -install_certificates(disk) +install_certificates(disk/certs) diskbuilder(disk) diff --git a/examples/http_client/disk/server.pem b/examples/http_client/disk/server.pem deleted file mode 100644 index 46190d9a4b..0000000000 --- a/examples/http_client/disk/server.pem +++ /dev/null @@ -1,83 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIJRAIBADANBgkqhkiG9w0BAQEFAASCCS4wggkqAgEAAoICAQDMVU8mqaDVEPFG -zIKj9kNXTQB8RDjhVFg6xHm+3gXcp0PXyOqbWjsk9ZExIwAARo29nqWokNmisV3u -euBoUh2Ne/giP9Ir+76PViR3C+xvBUkI1YdTu+HkmLQDJLPKlzgF99PKUUh9467a -H0jWix59NyPSy1J9OVv+Z4V3bdCuiaPJI2qv+ysANYOmPQYQPWAINEo3xmNjRkx5 -j1QWgdhV2cmw4YjRCWgFzhzpSQYq0mP3AgZJ0L+hPoc2KSth/wSkLRS3q+lJTFTA -RUtw7w7icFQwRMiMNSGKvJJ9FMgNUsiAA6Ju6iHnBj9qd2mpDlJ+pV+3j0fFw2nK -7bL7Pjs9N9ninZJbigmpTieKPOkVuTV3DNCmWAqu5QtHGjYa8ySvzjBPJT83c/WU -G4eB8eUMjeq36d6YXurCLa/Ext1KUQidgDFNtys7YqqvtRHL45qC18hzlgzWy4o5 -UIOolS3pAkRorop65UXIV/lyn+i/SrzTcidQEB4DYI9DBWI7hMFSp6YHEaLrYE8I -9eQGlIhjsFJQTQTKwdzA83812BHkfeH2j6wuWAOnzTfztoKVisqbHFoybTtPw1It -ELxlMUuVQGJnoftwBipcfljlO3ukLXszQspppCSWqFeXF58IErhvkM9nOTLS9FWU -uGihqHEUZuUdjQfkYMZQ33dvTwL5DQIDAQABAoICAFMRf2sVXNgh2iTRMW8cx4Wv -PTpHteQNGQ1TILdypWlt6ovIF3Buv0qoNo2GUBLMnATxgSFFOeUFPE2eUKq4b2E8 -e8TYs0XEQ1UcTRjElSsRN/7KaAingg8f8VaxzhHXZiQu+z/tyY28STJNs5vc1NlM -2Fhy5icYcYi8ZysFTugV9qrnphDDCFk+aDXl6xghC3BqaZ3BHliWlLJ6jVHVGbDV -KESre6aXr11kzKYeVM6F17Psc8gAgFc6B1gCZquhNC7WUOzWi7GLwYyk3yWbWi6+ -QcuZuvodA3TWcqs9DpM0C2TXlWqH7p5wEMnkn+TleK4Z5w7FxgTlC5Y/ChtP0l63 -ElVjkOggliKCv6xmCAkVoG/7KaiFaTZgPZ+Mpl0S2toTdgWDQ/YnKxoJqX9AK03k -Tt5NGm6vdjQbb7V84uV1N1vl/DQXpI7E6teQagRIgXlnhPDlU5pl90Nx7HziUbdT -bHQ2yfL18M8/Q42u5o/+5jfB78VGe1tW3pznoybbkGdOWuhCldSypOL9pXShmLK+ -wKKQZomvw2J7SGW4yFnYcQ1avaaImtx/YQuLp/VazjXXU5miJ6hyxrxKLoNmZry8 -v/a/gzxqaWe9LgomCHBimBpOG6uf5rCV+5lMEjKfP8ka6Su+ZR8TITDIcjCnpNQo -MukSIiE2UaMA3Y9RSYbhAoIBAQD7XhwQB02JGOUjpYkwIIZDJYadRxBRm1cCYbPS -KHqLjHyVLgf1Iix8TzbOXvuy6rp2nmaaNP2L4uxNphpe3rK5LquG1rD/QHNt2vQx -PmDMiUcExNfEJa+x77G6S3PDyN3/kThJAHUBrnQmViur/kHFBDQT5H48HfroHL7O -V3+0U0O4FwPg7xo6m3lkTSMgxvW57LBzcSfII5qLh1+weVHWN5EgmwK6wlwtSZte -uBEdckZG0D2gfzxEn9VxHjiEll0YQjatwZ6KNamnQmVbXS3AEh41B2YVa6uAWAvT -Jm2gmlrN/6G3tqaFZBa0ZNIh5WmvHRGtJsimNrk/1urw+qhpAoIBAQDQGU2QM6/z -XuhrjkhKV1KiAbQAfEzsPCua3hQNZN10hMqhgC3KM2rF08NE9QOQShmFDIyxMQ7n -7iMED9g7sgwdXRTPrkhRj5Kumw3Mxx9P7tOP7I3HgHCyJGrsOePArgEELomF47Ye -vlGIGOVJkGRkQKxH1GMFKTApQRobCsla9pFkoHwFuPvQbgIY5HSZHEFzkr42Z1me -/7P9sHlVpiJS2iSY9c/v1ijKoElANPhWSewHXU5JS/ykQ6RM4+1Ezz9eOYFB66nI -/64AYLtZ/12KfaU8v2nL3jRgWbekj60jwYwtvgStKq3bmoa77pWd13OEQoYQ1TWq -3yNf77CHklcFAoIBAQDlUzBq9x6nqwyxfr4wKBwVelDbgA/FQ+EXjSdO3hse4SZv -uzULUQggyOwJTuJ2kmtW+KBGo32Fd5t6I8X/M1D+XTOjqo6D8LreSFlr66rIL52Z -HjRzAKbPwQv2GAwfXEwcctJU15Pn2s6ggxEIssgyA+m3FEiOdBOKW19ARNd/Hk+z -zVt+hOT6/wi6lx0rNBjObL92Mo/GqaS/Etwb1jLxKnS+uGy/Eb6imnqi6W7D3JN8 -21K98BZ9zQwfTSDm5X89OHtF/lSzZs3GiG6L8fBhQKZKXtulQzuNtcXj9zUgxnF9 -anCReJNnMq78GNcUkYXfere7rpxz94z+Twa8nxGxAoIBAQCFKKvJKkncoU54BIOd -jmMB57UW2nMedyIv5wtNxs0uUgqXbT3ftScNj4PUDDRLkB7kVDNy7sFrB6bAYqMr -D7NQ1s8KVatRBV4G/JDICGKw81HNNsFCd52lYEhLAIAMO0vLMTwuwJc49O+W9tdB -S3Px8rBLBkyWcNSwkUfzJePSoRRIH6k8wDHiHYKaCxXRI5N+MNgmV9uxMk4qd1Is -zDIKciwY9LIp9hAEGKzGuR5vmL2/uKTvof/33JC14uNgtH0H/iQH2pS6WWEuPLel -zxI8HEovpQw9t+IsBTzNwGI6Dw9z9bNoW3cEnsy4VOLILaVtbGP9QXEl9cLdELMV -DT9BAoIBAQCi1cFk2C7EluvHClbzlbUQBsHpCchisLgLF/appYCMMlZ55Ys2iQ2b -Q8xm4iNbB7r+kyBv2R7xnCPIM5dsEhWHmv1mfQY+sGZPBneTaSPV/4ZLG7YueepR -A52j5A6uXOt4R8T0te8GhhE7C0VcJU/i0vZQoAAQ5WGEi4SnO9dFhRAlcp+siIC7 -giiMuBQ82p2DWl48hpWJI+pLT+TmseFFZQ26GdFuDzJQxGF4fmRM30GHGXiiGU7S -8a+vXhA/w8SAaZhr0WadaD4bl5LvJUBuM+XVWr+ftS1VwVTEfuIAe4dFgNaDbc4Z -ucqyshD/YlGrku77JOcDw8KEexz1MeXE ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIFXTCCA0WgAwIBAgIJAL40szehpk5dMA0GCSqGSIb3DQEBDQUAMEUxCzAJBgNV -BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX -aWRnaXRzIFB0eSBMdGQwHhcNMTgwMjA1MjM0ODM4WhcNMjgwMjAzMjM0ODM4WjBF -MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 -ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC -CgKCAgEAzFVPJqmg1RDxRsyCo/ZDV00AfEQ44VRYOsR5vt4F3KdD18jqm1o7JPWR -MSMAAEaNvZ6lqJDZorFd7nrgaFIdjXv4Ij/SK/u+j1YkdwvsbwVJCNWHU7vh5Ji0 -AySzypc4BffTylFIfeOu2h9I1osefTcj0stSfTlb/meFd23QromjySNqr/srADWD -pj0GED1gCDRKN8ZjY0ZMeY9UFoHYVdnJsOGI0QloBc4c6UkGKtJj9wIGSdC/oT6H -NikrYf8EpC0Ut6vpSUxUwEVLcO8O4nBUMETIjDUhirySfRTIDVLIgAOibuoh5wY/ -andpqQ5SfqVft49HxcNpyu2y+z47PTfZ4p2SW4oJqU4nijzpFbk1dwzQplgKruUL -Rxo2GvMkr84wTyU/N3P1lBuHgfHlDI3qt+nemF7qwi2vxMbdSlEInYAxTbcrO2Kq -r7URy+OagtfIc5YM1suKOVCDqJUt6QJEaK6KeuVFyFf5cp/ov0q803InUBAeA2CP -QwViO4TBUqemBxGi62BPCPXkBpSIY7BSUE0EysHcwPN/NdgR5H3h9o+sLlgDp803 -87aClYrKmxxaMm07T8NSLRC8ZTFLlUBiZ6H7cAYqXH5Y5Tt7pC17M0LKaaQklqhX -lxefCBK4b5DPZzky0vRVlLhooahxFGblHY0H5GDGUN93b08C+Q0CAwEAAaNQME4w -HQYDVR0OBBYEFCS1cTijfysZ+WN0N67mR62VPezfMB8GA1UdIwQYMBaAFCS1cTij -fysZ+WN0N67mR62VPezfMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQENBQADggIB -AK3P0IurEm/AxIFrcbr4z1VVemZm/aMp2kU/x6UOKxnBh/IqrcCdYb5Ompr8NIML -dsra8+kaQD++sUNSyQfcYUTBmjatR756NfDSSSavywbutjTFM/0a1mID9xo4OaeQ -FoDI8OzU7rtxPWk+bn/C+vt1a0RzgE2xFFS1wxZoQ8UrLLdSoWAQMOVsk6H94H+j -kTZDRhYdC9Ij4Lo1zAPNH6BJT0AXFUZluteGsJlRG1y415mCTtPPogvXH88bLM2C -p1o0kMhrfXbJEQ3YHUbDjQ+E6vrTEe81Ym7RF2puSFakQAMF6USsHvQJY/AH2Ss1 -5amuGzu4Z1yLXY5Ln8uha8uTPHiMe0l82GNeBQZN3tv0b8npCP+8RJ1oBOoKurJJ -zVGemX1e61GHnK6L6mfZ78kYrbmc6O3IAGd4zEXRXKtIVyOhbVRrXzCA+NChMUdB -jX1QOCk5n9pJFELsgEseJLqqraRhtoqeQnMZ+F9c/J07LAovVdq7JHhM5DZj5RDr -KzIcu1456OLZWkIqt3gK2UoGL4KjzIju81n4PskjQ4L0QJQlTt1qVGU/lyBT+0vU -dRHGIRJ/IS1urrN/+w6XDiYN5Wynoo+FQFLcTRKsJuG8xzCJ1YcOn0fkkl7sqHTU -IITTjSEm0Cj8updtdoNrgNbedscZA5MxmgmPQZTzfcfM ------END CERTIFICATE----- diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index f07a19ee6a..f1bdf31d0b 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -24,11 +24,11 @@ static SSL_CTX* init_ssl_context() disk.init_fs([] (fs::error_t err, auto& fs) { assert(!err); - err = fs.print_subtree("/cert_bundle"); + err = fs.print_subtree("/certs"); assert(err == fs::no_error && "Need certificate bundle folder present"); }); - auto ents = disk.fs().ls("/cert_bundle"); + auto ents = disk.fs().ls("/certs"); // initialize client context openssl::init(); return openssl::create_client(ents, true); @@ -42,7 +42,7 @@ static SSL_CTX* init_ssl_context() static void begin_http(net::Inet& inet) { using namespace http; - /* + static Basic_client basic{inet.tcp()}; const std::string url{"http://www.google.com"}; @@ -58,7 +58,7 @@ static void begin_http(net::Inet& inet) printf("Make sure the virtual machine can reach internet.\n"); } }); - */ + auto* ctx = init_ssl_context(); assert(ctx != nullptr); From 0eca5e4a08ebcfaa8426663079607622c4a88d4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 20 Apr 2018 15:22:02 +0200 Subject: [PATCH 215/723] uplink: Force scheme prefix in url, added verify opt, use wss scheme when secure --- lib/uplink/config.cpp | 8 ++++++ lib/uplink/config.hpp | 4 ++- lib/uplink/starbase/CMakeLists.txt | 3 +++ lib/uplink/starbase/config.json | 2 +- lib/uplink/ws_uplink.cpp | 40 ++++++++++++++++-------------- 5 files changed, 36 insertions(+), 21 deletions(-) diff --git a/lib/uplink/config.cpp b/lib/uplink/config.cpp index 77026e8230..e0e9511f74 100644 --- a/lib/uplink/config.cpp +++ b/lib/uplink/config.cpp @@ -58,6 +58,9 @@ namespace uplink { config_.url = cfg["url"].GetString(); config_.token = cfg["token"].GetString(); + Expects(config_.url.is_valid() && "Invalid URL given (may have missing scheme)"); + Expects(not config_.url.scheme().empty() && "Scheme missing in URL (http or https)"); + // Decide stack/interface if(cfg.HasMember("index")) { @@ -106,6 +109,11 @@ namespace uplink { config_.certs_path = cfg["certs"].GetString(); } + if(cfg.HasMember("verify")) + { + config_.verify_certs = cfg["verify"].GetBool(); + } + return config_; } diff --git a/lib/uplink/config.hpp b/lib/uplink/config.hpp index ea6d5c0084..888f20a743 100644 --- a/lib/uplink/config.hpp +++ b/lib/uplink/config.hpp @@ -22,6 +22,7 @@ #include #include #include +#include namespace uplink { @@ -30,10 +31,11 @@ namespace uplink { struct Config { net::Inet* inet; - std::string url; + uri::URI url; std::string token; std::string tag; std::string certs_path = default_cert_path; + bool verify_certs = true; bool reboot = true; bool ws_logging = true; bool serialize_ct = false; diff --git a/lib/uplink/starbase/CMakeLists.txt b/lib/uplink/starbase/CMakeLists.txt index 54f5d04538..58c5847ed9 100644 --- a/lib/uplink/starbase/CMakeLists.txt +++ b/lib/uplink/starbase/CMakeLists.txt @@ -62,3 +62,6 @@ set(LIBRARIES # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) + +#install_certificates(disk/certs) +#diskbuilder(disk) diff --git a/lib/uplink/starbase/config.json b/lib/uplink/starbase/config.json index 7c59169931..4ee79c3211 100644 --- a/lib/uplink/starbase/config.json +++ b/lib/uplink/starbase/config.json @@ -9,7 +9,7 @@ } ], "uplink" : { - "url" : "10.0.0.1:9090", + "url" : "http://10.0.0.1:9090", "token" : "kappa123", "reboot" : true } diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index e4d0593909..9ba8f1a0e4 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -43,9 +43,11 @@ #include #include -static SSL_CTX* init_ssl_context(const std::string& certs_path) +static SSL_CTX* init_ssl_context(const std::string& certs_path, bool verify) { - MYINFO("Reading certificates from disk @ %s", certs_path.c_str()); + MYINFO("Reading certificates from disk @ %s. Verify certs: %s", + certs_path.c_str(), verify ? "YES" : "NO"); + auto& disk = fs::memdisk(); disk.init_fs([] (auto err, auto&) { Ensures(!err && "Error init filesystem"); @@ -66,7 +68,7 @@ static SSL_CTX* init_ssl_context(const std::string& certs_path) // initialize client context openssl::init(); - return openssl::create_client(ents, true); + return openssl::create_client(ents, verify); } namespace uplink { @@ -117,15 +119,10 @@ namespace uplink { inet.ifname().c_str(), id_.c_str()); Expects(inet.ip_addr() != 0 && "Network interface not configured"); - Expects(not config_.url.empty()); - - const uri::URI url{config_.url}; - Expects(url.is_valid() && "Invalid URL"); - - if(url.scheme() == "https") + if(config_.url.scheme_is_secure()) { - auto* ssl_context = init_ssl_context(config_.certs_path); + auto* ssl_context = init_ssl_context(config_.certs_path, config_.verify_certs); Expects(ssl_context != nullptr && "Secure URL given but no valid certificates found"); client_ = std::make_unique(inet.tcp(), ssl_context, @@ -167,14 +164,14 @@ namespace uplink { void WS_uplink::auth() { - std::string url{"http://"}; - url.append(config_.url).append("/auth"); + const static std::string endpoint{"/auth"}; - //static const std::string auth_data{"{ \"id\": \"testor\", \"key\": \"kappa123\"}"}; + uri::URI url{config_.url}; + url << endpoint; - MYINFO("[ %s ] Sending auth request to %s", isotime::now().c_str(), url.c_str()); + MYINFO("[ %s ] Sending auth request to %s", isotime::now().c_str(), url.to_string().c_str()); - client_->post(http::URI{url}, + client_->post(url, { {"Content-Type", "application/json"} }, auth_data(), {this, &WS_uplink::handle_auth_response}, @@ -220,12 +217,17 @@ namespace uplink { { Expects(not token_.empty() and client_ != nullptr); - std::string url{"ws://"}; - url.append(config_.url).append("/dock"); + const static std::string endpoint{"/dock"}; + + // for now, build the websocket url based on the auth url. + // maybe this will change in the future, and the ws url will have it's own + // entry in the config + std::string scheme = (config_.url.scheme_is_secure()) ? "wss://" : "ws://"; + uri::URI url{scheme + config_.url.host_and_port() + endpoint}; - MYINFO("[ %s ] Dock attempt to %s", isotime::now().c_str(), url.c_str()); + MYINFO("[ %s ] Dock attempt to %s", isotime::now().c_str(), url.to_string().c_str()); - net::WebSocket::connect(*client_, http::URI{url}, {this, &WS_uplink::establish_ws}); + net::WebSocket::connect(*client_, url, {this, &WS_uplink::establish_ws}); } void WS_uplink::establish_ws(net::WebSocket_ptr ws) From 870bda9197584a9e79365d52973e34bd99c3bbf2 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 23 Apr 2018 13:42:11 +0200 Subject: [PATCH 216/723] tcp: Use RTC instead of OS nano timestamp --- src/net/tcp/tcp.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index 0793a67380..3ce78af747 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -22,7 +22,7 @@ #include #include // checksum #include -#include // nanos_since_boot (get_ts_value) +#include // nanos_now (get_ts_value) using namespace std; using namespace net; @@ -365,7 +365,7 @@ seq_t TCP::generate_iss() { uint32_t TCP::get_ts_value() const { - return ((OS::nanos_since_boot() / 1000000000ull) & 0xffffffff); + return ((RTC::nanos_now() / 1000000000ull) & 0xffffffff); } void TCP::drop(const tcp::Packet&) { From fdf99a6de6660258e31ec19c49c96e2290543224 Mon Sep 17 00:00:00 2001 From: nikhilap1 Date: Tue, 10 Apr 2018 01:46:08 +0530 Subject: [PATCH 217/723] First IPv6 commit --- api/kernel/terminal.hpp | 2 +- api/net/dns/client.hpp | 3 +- api/net/http/basic_client.hpp | 1 + api/net/http/server.hpp | 2 +- api/net/inet.hpp | 544 ++++++++++++------ api/net/inet4 | 7 - api/net/inet4.hpp | 404 ------------- api/net/inet_common.hpp | 16 +- api/net/ip4/ip4.hpp | 27 +- api/net/ip4/packet_icmp4.hpp | 2 +- api/net/ip4/udp.hpp | 22 +- api/net/ip6/addr.hpp | 209 +++++++ api/net/ip6/header.hpp | 67 +++ api/net/ip6/icmp6.hpp | 76 ++- api/net/ip6/ip6.hpp | 458 ++++++--------- api/net/ip6/packet_ip6.hpp | 196 +++++-- api/net/ip6/udp6.hpp | 139 +---- api/net/nat/napt.hpp | 3 +- api/net/netfilter.hpp | 7 +- api/net/openssl/tls_stream.hpp | 2 +- api/net/router | 2 +- api/net/router.hpp | 4 +- api/net/super_stack.hpp | 34 +- api/net/tcp/tcp.hpp | 17 +- api/net/vlan | 4 +- api/plugins/unik.hpp | 4 +- api/util/async.hpp | 2 +- examples/IRCd/ircd/ircd.hpp | 2 +- examples/IRCd/service.cpp | 2 +- examples/LiveUpdate/service.cpp | 2 +- examples/TCP_perf/service.cpp | 2 +- examples/TLS_server/service.cpp | 2 +- examples/UDP_perf/service.cpp | 2 +- examples/acorn/service.cpp | 2 +- examples/demo_linux/service.cpp | 2 +- examples/demo_service/service.cpp | 2 +- examples/mender/service.cpp | 4 +- examples/scoped_profiler/service.cpp | 2 +- examples/syslog/service.cpp | 6 +- examples/tcp/service.cpp | 2 +- examples/websocket/service.cpp | 4 +- lib/LiveUpdate/serialize_tcp.cpp | 4 +- lib/mana/examples/simple/service.cpp | 4 +- lib/microLB/micro_lb/autoconf.cpp | 4 +- lib/microLB/micro_lb/balancer.hpp | 4 +- lib/uplink/config.cpp | 7 +- lib/uplink/config.hpp | 3 +- lib/uplink/starbase/service.cpp | 2 +- lib/uplink/ws_uplink.cpp | 2 +- lib/uplink/ws_uplink.hpp | 6 +- src/CMakeLists.txt | 3 +- src/include/sockfd.hpp | 2 +- src/include/udp_fd.hpp | 2 +- src/kernel/terminal.cpp | 3 + src/net/configure.cpp | 5 +- src/net/dhcp/dh4client.cpp | 1 + src/net/dhcp/dhcpd.cpp | 1 + src/net/dns/client.cpp | 1 + src/net/http/server.cpp | 1 + src/net/{inet4.cpp => inet.cpp} | 87 ++- src/net/ip4/arp.cpp | 2 +- src/net/ip4/icmp4.cpp | 3 +- src/net/ip4/ip4.cpp | 11 + src/net/ip4/udp.cpp | 16 + src/net/ip6/ip6.cpp | 402 +++++++++---- src/net/nat/napt.cpp | 1 + src/net/super_stack.cpp | 27 +- src/net/tcp/tcp.cpp | 13 + src/plugins/terminal.cpp | 6 +- src/plugins/unik.cpp | 10 +- src/posix/tcp_fd.cpp | 2 +- src/posix/udp_fd.cpp | 4 +- src/util/syslog_facility.cpp | 6 +- test/CMakeLists.txt | 4 +- .../kernel/integration/LiveUpdate/service.cpp | 4 +- .../integration/modules/mod2/service.cpp | 4 +- test/kernel/integration/term/service.cpp | 4 +- test/linux/router/async_device.hpp | 2 +- test/linux/router/nacl.cpp | 6 +- test/linux/router/service.cpp | 2 +- test/linux/tcp/service.cpp | 6 +- test/net/integration/configure/service.cpp | 1 + test/net/integration/dhclient/service.cpp | 6 +- test/net/integration/dhcpd/service.cpp | 16 +- .../dhcpd_dhclient_linux/service.cpp | 4 +- test/net/integration/dns/service.cpp | 4 +- test/net/integration/gateway/service.cpp | 10 +- test/net/integration/http/service.cpp | 4 +- test/net/integration/icmp/service.cpp | 4 +- test/net/integration/ipv6/CMakeLists.txt | 33 ++ test/net/integration/ipv6/service.cpp | 33 ++ test/net/integration/ipv6/vm.json | 6 + test/net/integration/nat/service.cpp | 24 +- test/net/integration/router/service.cpp | 10 +- test/net/integration/tcp/service.cpp | 24 +- test/net/integration/transmit/service.cpp | 4 +- test/net/integration/udp/service.cpp | 4 +- test/net/integration/vlan/service.cpp | 14 +- test/net/integration/websocket/service.cpp | 4 +- test/net/unit/error.cpp | 2 +- test/net/unit/ip4.cpp | 4 +- test/net/unit/ip6.cpp | 47 ++ test/net/unit/napt_test.cpp | 4 +- test/net/unit/path_mtu_discovery.cpp | 30 +- test/net/unit/router_test.cpp | 18 +- test/net/unit/super_stack.cpp | 24 +- test/plugin/integration/unik/service.cpp | 12 +- .../integration/syslog_plugin/service.cpp | 4 +- test/posix/integration/udp/service.cpp | 4 +- test/stress/service.cpp | 4 +- .../tar/tar_example/l1_f1/l2/service.cpp | 6 +- .../tar/tar_example/l1_f1/service.cpp | 6 +- .../tar_gz/tar_example/l1_f1/l2/service.cpp | 6 +- .../tar_gz/tar_example/l1_f1/service.cpp | 6 +- 114 files changed, 1866 insertions(+), 1483 deletions(-) delete mode 100644 api/net/inet4 delete mode 100644 api/net/inet4.hpp create mode 100644 api/net/ip6/addr.hpp create mode 100644 api/net/ip6/header.hpp rename src/net/{inet4.cpp => inet.cpp} (73%) create mode 100644 test/net/integration/ipv6/CMakeLists.txt create mode 100644 test/net/integration/ipv6/service.cpp create mode 100644 test/net/integration/ipv6/vm.json create mode 100644 test/net/unit/ip6.cpp diff --git a/api/kernel/terminal.hpp b/api/kernel/terminal.hpp index c72cd35281..d7645ffc8a 100644 --- a/api/kernel/terminal.hpp +++ b/api/kernel/terminal.hpp @@ -20,7 +20,7 @@ #define API_KERNEL_TERMINAL_HPP #include -#include +#include #include #include #include diff --git a/api/net/dns/client.hpp b/api/net/dns/client.hpp index c5896c828f..4fbb5d99ef 100644 --- a/api/net/dns/client.hpp +++ b/api/net/dns/client.hpp @@ -18,7 +18,6 @@ #ifndef NET_DNS_CLIENT_HPP #define NET_DNS_CLIENT_HPP -#include #include #include #include @@ -38,7 +37,7 @@ namespace net { public: using Stack = IP4::Stack; - using Resolve_handler = Stack::resolve_func; + using Resolve_handler = IP4::resolve_func; using Address = ip4::Addr; using Hostname = std::string; using timestamp_t = RTC::timestamp_t; diff --git a/api/net/http/basic_client.hpp b/api/net/http/basic_client.hpp index a8f3e35594..17fec35dff 100644 --- a/api/net/http/basic_client.hpp +++ b/api/net/http/basic_client.hpp @@ -23,6 +23,7 @@ #include "client_connection.hpp" #include +#include #include #include diff --git a/api/net/http/server.hpp b/api/net/http/server.hpp index e79efc7d5c..dd1363f5c3 100644 --- a/api/net/http/server.hpp +++ b/api/net/http/server.hpp @@ -184,7 +184,7 @@ namespace http { * * Usage example: * @code - * auto& inet = net::Inet4::stack<0>(); + * auto& inet = net::Inet::stack<0>(); * Expects(inet.is_configured()); * auto server = http::make_server(inet.tcp()); * server->on_request([](auto req, auto rw){...}); diff --git a/api/net/inet.hpp b/api/net/inet.hpp index 14a37fb734..3a959d5ffb 100644 --- a/api/net/inet.hpp +++ b/api/net/inet.hpp @@ -18,6 +18,7 @@ #ifndef NET_INET_HPP #define NET_INET_HPP +#include #include #include #include @@ -27,142 +28,100 @@ #include #include "conntrack.hpp" +#include "ip4/ip4.hpp" +#include "ip4/udp.hpp" +#include "ip4/icmp4.hpp" +#include "ip4/arp.hpp" +#include "ip6/ip6.hpp" +#include "dns/client.hpp" +#include "tcp/tcp.hpp" +#include "super_stack.hpp" + namespace net { + class Arp; class TCP; class UDP; - class DHClient; class ICMPv4; + class DHClient; + class Super_stack; + + /** A complete IP network stack */ + class Inet { + public: - /** - * An abstract IP-stack interface. - * Provides a common interface for IPv4 and (future) IPv6, simplified with - * no constructors etc. - **/ - template - struct Inet { - using Stack = Inet; - using IP_packet_ptr = typename IPV::IP_packet_ptr; - using IP_addr = typename IPV::addr; + using Stack = class Inet; + using IP_packet_ptr = IP4::IP_packet_ptr; + using IP_addr = IP4::addr; using Forward_delg = delegate; using Route_checker = delegate; using IP_packet_factory = delegate; - template - using resolve_func = delegate; - - using Vip_list = std::vector; - using Port_utils = std::map; - - - /// - /// NETWORK CONFIGURATION - /// - - /** Get IP address of this interface **/ - virtual IP_addr ip_addr() const = 0; - - /** Get netmask of this interface **/ - virtual IP_addr netmask() const = 0; - - /** Get default gateway for this interface **/ - virtual IP_addr gateway() const = 0; - - /** Get default dns for this interface **/ - virtual IP_addr dns_addr() const = 0; - - /** Get broadcast address for this interface **/ - virtual IP_addr broadcast_addr() const = 0; - - /** Set default gateway for this interface */ - virtual void set_gateway(IP_addr server) = 0; - - /** Set DNS server for this interface */ - virtual void set_dns_server(IP_addr server) = 0; - - /** Configure network for this interface */ - virtual void network_config(IP_addr ip, - IP_addr nmask, - IP_addr gateway, - IP_addr dnssrv = IPV::ADDR_ANY) = 0; - - /** Reset network configuration for this interface */ - virtual void reset_config() = 0; - + using resolve_func = delegate; + using on_configured_func = delegate; using dhcp_timeout_func = delegate; - /** Use DHCP to configure this interface */ - virtual void negotiate_dhcp(double timeout = 10.0, dhcp_timeout_func = nullptr) = 0; - - virtual bool is_configured() const = 0; - - using on_configured_func = delegate; + using Port_utils = std::map; + using Vip4_list = std::vector; + using Vip6_list = std::vector; - /** Assign callback to when the stack has been configured */ - virtual void on_config(on_configured_func handler) = 0; + std::string ifname() + { return nic_.device_name(); } - /** Get a list of virtual IP4 addresses assigned to this interface */ - virtual const Vip_list virtual_ips() const = 0; + MAC::Addr link_addr() + { return nic_.mac(); } - /** Check if an IP is a (possibly virtual) loopback address */ - virtual bool is_loopback(IP_addr a) const = 0; + hw::Nic& nic() + { return nic_; } - /** Add an IP address as a virtual loopback IP */ - virtual void add_vip(IP_addr a) = 0; + IP4::addr ip_addr() + { return ip4_addr_; } - /** Remove an IP address from the virtual loopback IP list */ - virtual void remove_vip(IP_addr a) = 0; + IP4::addr netmask() + { return netmask_; } - /** Determine the appropriate source address for a destination. */ - virtual IP_addr get_source_addr(IP_addr dest) = 0; + IP4::addr gateway() + { return gateway_; } - /** Determine if an IP address is a valid source address for this stack */ - virtual bool is_valid_source(IP_addr) const = 0; + IP4::addr dns_addr() + { return dns_server_; } + IP4::addr broadcast_addr() + { return ip4_addr_ | ( ~ netmask_); } - /** - * @brief Get the active conntrack for this stack. - * - * @return The conntrack for this stack, nullptr if none. - */ - virtual std::shared_ptr& conntrack() = 0; - virtual const std::shared_ptr& conntrack() const = 0; + IP6::addr ip6_addr() + { return ip6_addr_; } - /** - * @brief Enables the conntrack for this stack, - * setting up the necessary hooks. - * - * @param The conntrack to assign to this stack. - */ - virtual void enable_conntrack(std::shared_ptr) = 0; + IP6::addr netmask6() + { return ip6_prefix_; } + IP6::addr gateway6() + { return ip6_gateway_; } - /// - /// PROTOCOL OBJECTS - /// + void cache_link_addr(IP4::addr ip, MAC::Addr mac); + void flush_link_cache(); + void set_link_cache_flush_interval(std::chrono::minutes min); - /** Get the IP protocol object for this interface */ - virtual IPV& ip_obj() = 0; + /** Get the IP-object belonging to this stack */ + IP4& ip_obj() + { return ip4_; } - /** Get the TCP protocol object for this interface */ - virtual TCP& tcp() = 0; + /** Get the IP6-object belonging to this stack */ + IP6& ip6_obj() + { return ip6_; } - /** Get the UDP protocol object for this interface */ - virtual UDP& udp() = 0; + /** Get the TCP-object belonging to this stack */ + TCP& tcp() { return tcp_; } - /** Get the ICMP protocol object for this interface */ - virtual ICMPv4& icmp() = 0; + /** Get the UDP-object belonging to this stack */ + UDP& udp() { return udp_; } - virtual Port_utils& tcp_ports() = 0; - virtual Port_utils& udp_ports() = 0; + /** Get the ICMP-object belonging to this stack */ + ICMPv4& icmp() { return icmp_; } - /// - /// PATH MTU DISCOVERY - /// PACKETIZATION LAYER PATH MTU DISCOVERY - /// ERROR REPORTING - /// Communication from ICMP and IP to UDP and TCP - /// + /** Get the DHCP client (if any) */ + auto dhclient() { return dhcp_; } /** * @brief Disable or enable Path MTU Discovery (enabled by default) @@ -176,7 +135,8 @@ namespace net { * This could be set to "infinity" (PMTU should never be * increased) by setting the value to IP4::INFINITY */ - virtual void set_path_mtu_discovery(bool on, uint16_t aged = 10) = 0; + void set_path_mtu_discovery(bool on, uint16_t aged = 10) + { ip4_.set_path_mtu_discovery(on, aged); } /** * @brief Triggered by IP when a Path MTU value has grown stale and the value @@ -187,7 +147,7 @@ namespace net { * @param[in] dest The destination/path * @param[in] pmtu The reset PMTU value */ - virtual void reset_pmtu(Socket dest, typename IPV::PMTU pmtu) = 0; + void reset_pmtu(Socket dest, IP4::PMTU pmtu); /** * Error reporting @@ -198,109 +158,337 @@ namespace net { * * Forwards errors to the transport layer (UDP and TCP) */ - virtual void error_report(Error& err, Packet_ptr orig_pckt) = 0; + void error_report(Error& err, Packet_ptr orig_pckt); - /// - /// DNS - /// + /** + * Set the forwarding delegate used by this stack. + * If set it will get all incoming packets not intended for this stack. + * NOTE: This delegate is expected to call the forward chain + */ + void set_forward_delg(Forward_delg fwd) { + ip4_.set_packet_forwarding(fwd); + } - /** DNS resolution */ - virtual void resolve(const std::string& hostname, - resolve_func func, - bool force = false) = 0; - virtual void resolve(const std::string& hostname, - IP_addr server, - resolve_func func, - bool force = false) = 0; + /** + * Assign a delegate that checks if we have a route to a given IP + */ + void set_route_checker(Route_checker delg); - virtual void set_domain_name(std::string domain_name) = 0; + /** + * Get the forwarding delegate used by this stack. + */ + Forward_delg forward_delg() + { return ip4_.forward_delg(); } - virtual const std::string& domain_name() const = 0; - /// - /// LINK LAYER - /// + Packet_ptr create_packet() { + return nic_.create_packet(nic_.frame_offset_link()); + } - /** Get the network interface device */ - virtual hw::Nic& nic() = 0; + /** + * Provision an IP packet + * @param proto : IANA protocol number. + */ + IP4::IP_packet_ptr create_ip_packet(Protocol proto) { + auto raw = nic_.create_packet(nic_.frame_offset_link()); + auto ip_packet = static_unique_ptr_cast(std::move(raw)); - /** Get interface name for this interface **/ - virtual std::string ifname() const = 0; + ip_packet->init(proto); - /** Get linklayer address for this interface **/ - virtual MAC::Addr link_addr() const = 0; + return ip_packet; + } - /** Add cache entry to the link / IP address cache */ - virtual void cache_link_addr(IP_addr, MAC::Addr) = 0; + IP6::IP_packet_ptr create_ip6_packet(Protocol proto) { + auto raw = nic_.create_packet(nic_.frame_offset_link()); + auto ip_packet = static_unique_ptr_cast(std::move(raw)); - /** Flush link / IP address cache */ - virtual void flush_link_cache() = 0; + ip_packet->init(proto); - /** Set the regular interval for link address cache flushing */ - virtual void set_link_cache_flush_interval(std::chrono::minutes) = 0; + return ip_packet; + } + IP_packet_factory ip_packet_factory() + { return IP_packet_factory{this, &Inet::create_ip_packet}; } - /// - /// ROUTING - /// + /** MTU retreived from Nic on construction */ + uint16_t MTU() + { return MTU_; } - /** Set an IP forwarding delegate. E.g. used to enable routing. - * NOTE: The packet forwarder is expected to call the forward_chain + /** + * @func a delegate that provides a hostname and its address, which is 0 if the + * name @hostname was not found. Note: Test with INADDR_ANY for a 0-address. **/ - virtual void set_forward_delg(Forward_delg) = 0; - - /** Assign boolean function to determine if we have route to a given IP */ - virtual void set_route_checker(Route_checker) = 0; + void resolve(const std::string& hostname, + resolve_func func, + bool force = false); - /** Get the IP forwarding delegate */ - virtual Forward_delg forward_delg() = 0; + void resolve(const std::string& hostname, + IP4::addr server, + resolve_func func, + bool force = false); + void set_domain_name(std::string domain_name) + { this->domain_name_ = std::move(domain_name); } - /// - /// PACKET MANAGEMENT - /// + const std::string& domain_name() + { return this->domain_name_; } - /** Get Maximum Transmission Unit **/ - virtual uint16_t MTU() const = 0; + void set_gateway(IP4::addr gateway) + { + this->gateway_ = gateway; + } - /** Provision empty anonymous packet **/ - virtual Packet_ptr create_packet() = 0; + void set_dns_server(IP4::addr server) + { + this->dns_server_ = server; + } - /** Delegate to provision initialized IP packet **/ - virtual IP_packet_factory ip_packet_factory() = 0; + /** + * @brief Try to negotiate DHCP + * @details Initialize DHClient if not present and tries to negotitate dhcp. + * Also takes an optional timeout parameter and optional timeout function. + * + * @param timeout number of seconds before request should timeout + * @param dhcp_timeout_func DHCP timeout handler + */ + void negotiate_dhcp(double timeout = 10.0, dhcp_timeout_func = nullptr); + + bool is_configured() + { + return ip4_addr_ != 0; + } + + bool is_configured_v6() + { + return ip6_addr_ != IP6::ADDR_ANY; + } + + // handler called after the network is configured, + // either by DHCP or static network configuration + void on_config(on_configured_func handler) + { + configured_handlers_.push_back(handler); + } + + /** We don't want to copy or move an IP-stack. It's tied to a device. */ + Inet(Inet&) = delete; + Inet(Inet&&) = delete; + Inet& operator=(Inet) = delete; + Inet operator=(Inet&&) = delete; + + void network_config(IP4::addr addr, + IP4::addr nmask, + IP4::addr gateway, + IP4::addr dns = IP4::ADDR_ANY, + IP6::addr addr6 = IP6::ADDR_ANY, + IP6::addr prefix6 = IP6::ADDR_ANY, + IP6::addr gateway6 = IP6::ADDR_ANY); + + virtual void + reset_config() + { + this->ip4_addr_ = IP4::ADDR_ANY; + this->gateway_ = IP4::ADDR_ANY; + this->netmask_ = IP4::ADDR_ANY; + this->ip6_addr_ = IP6::ADDR_ANY; + this->ip6_gateway_ = IP6::ADDR_ANY; + this->ip6_prefix_ = IP6::ADDR_ANY; + } + + // register a callback for receiving signal on free packet-buffers + virtual void + on_transmit_queue_available(transmit_avail_delg del) { + tqa.push_back(del); + } + + size_t transmit_queue_available() { + return nic_.transmit_queue_available(); + } + + size_t buffers_available() { + return nic_.buffers_available(); + } + size_t buffers_total() { + return nic_.buffers_total(); + } + + void force_start_send_queues(); + + void move_to_this_cpu(); + + int get_cpu_id() const noexcept { + return this->cpu_id; + } + + /** Return the stack on the given Nic */ + template + static auto&& stack() + { + return Super_stack::get(N); + } + + /** Static IP config */ + template + static auto&& ifconfig( + IP4::addr addr, + IP4::addr nmask, + IP4::addr gateway, + IP4::addr dns = IP4::ADDR_ANY) + { + stack().network_config(addr, nmask, gateway, dns); + return stack(); + } + + /** DHCP config */ + template + static auto& ifconfig(double timeout = 10.0, dhcp_timeout_func on_timeout = nullptr) + { + if (timeout > 0.0) + stack().negotiate_dhcp(timeout, on_timeout); + return stack(); + } + + const Vip4_list virtual_ips() const noexcept + { return vip4s_; } + + const Vip6_list virtual_ips() const noexcept + { return vip6s_; } + + /** Check if IP4 address is virtual loopback */ + bool is_loopback(IP4::addr a) + { + return a.is_loopback() + or std::find( vip4s_.begin(), vip4s_.end(), a) != vip4s_.end(); + } + + /** Check if IP6 address is virtual loopback */ + bool is_loopback(IP6::addr a) + { + return a.is_loopback() + or std::find( vip6s_.begin(), vip6s_.end(), a) != vip6s_.end(); + } + + /** add ip address as virtual loopback */ + void add_vip(ip4::addr a) + { + if (not is_loopback(a)) { + info("inet", "adding virtual ip address %s", a.to_string().c_str()); + vip4s_.emplace_back(a); + } + } + + void add_vip(ip6::addr a) + { + if (not is_loopback(a)) { + info("inet", "adding virtual ip6 address %s", a.to_string().c_str()); + vip6s_.emplace_back(a); + } + } + + /** Remove IP address as virtual loopback */ + void remove_vip(IP4::addr a) + { + auto it = std::find(vip4s_.begin(), vip4s_.end(), a); + if (it != vip4s_.end()) + vip4s_.erase(it); + } + + void remove_vip(IP6::addr a) + { + auto it = std::find(vip6s_.begin(), vip6s_.end(), a); + if (it != vip6s_.end()) + vip6s_.erase(it); + } + + IP4::addr get_source_addr(IP4::addr dest) + { + + if (dest.is_loopback()) + return {127,0,0,1}; + + if (is_loopback(dest)) + return dest; + + return ip_addr(); + } + + IP6::addr get_source_addr(IP6::addr dest) + { + + if (dest.is_loopback()) + return {0,0,0,1}; + + if (is_loopback(dest)) + return dest; + + return ip_addr(); + } + + bool is_valid_source(IP4::addr src) + { return src == ip_addr() or is_loopback(src); } + + bool is_valid_source(IP6::addr src) + { return src == ip6_addr() or is_loopback(src); } + + std::shared_ptr& conntrack() + { return conntrack_; } + + void enable_conntrack(std::shared_ptr ct); + + Port_utils& tcp_ports() + { return tcp_ports_; } + + Port_utils& udp_ports() + { return udp_ports_; } + + /** Initialize with ANY_ADDR */ + Inet(hw::Nic& nic); - /** Provision empty IP packet **/ - virtual IP_packet_ptr create_ip_packet(Protocol) = 0; + private: - /** Event triggered when there are available buffers in the transmit queue */ - virtual void on_transmit_queue_available(transmit_avail_delg del) = 0; + void process_sendq(size_t); + // delegates registered to get signalled about free packets + std::vector tqa; - /** Number of packets the transmit queue has room for */ - virtual size_t transmit_queue_available() = 0; + IP4::addr ip4_addr_; + IP4::addr netmask_; + IP4::addr gateway_; + IP4::addr dns_server_; + + IP6::addr ip6_addr_; + IP6::addr ip6_gateway_; + IP6::addr ip6_prefix_; - /** Number of buffers available in the bufstore */ - virtual size_t buffers_available() = 0; + Vip4_list vip4s_ = {{127,0,0,1}}; + Vip6_list vip6s_ = {{IP6::ADDR_LOOPBACK}}; - /** Number of total buffers in the bufstore */ - virtual size_t buffers_total() = 0; + // This is the actual stack + hw::Nic& nic_; + Arp arp_; + IP4 ip4_; + IP6 ip6_; + ICMPv4 icmp_; + UDP udp_; + TCP tcp_; - /** Start TCP (e.g. after system suspension). */ - virtual void force_start_send_queues() = 0; + Port_utils tcp_ports_; + Port_utils udp_ports_; + std::shared_ptr conntrack_; - /// - /// SMP - /// + // we need this to store the cache per-stack + DNSClient dns_; + std::string domain_name_; - /** Move this interface to the CPU executing the call */ - virtual void move_to_this_cpu() = 0; - virtual int get_cpu_id() const noexcept = 0; + std::shared_ptr dhcp_{}; + std::vector configured_handlers_; - /** Empty virtual destructor for Inet base **/ - virtual ~Inet() {} + int cpu_id; + const uint16_t MTU_; - }; //< class Inet -} //< namespace net + friend class Super_stack; + }; +} #endif diff --git a/api/net/inet4 b/api/net/inet4 deleted file mode 100644 index 2c0fb4e6f4..0000000000 --- a/api/net/inet4 +++ /dev/null @@ -1,7 +0,0 @@ -#ifndef INET4 -#define INET4 - -#include - - -#endif \ No newline at end of file diff --git a/api/net/inet4.hpp b/api/net/inet4.hpp deleted file mode 100644 index 77465fe4a7..0000000000 --- a/api/net/inet4.hpp +++ /dev/null @@ -1,404 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef NET_INET4_HPP -#define NET_INET4_HPP - -#include - -#include "inet.hpp" -#include "ip4/arp.hpp" -#include "ip4/ip4.hpp" -#include "ip4/udp.hpp" -#include "ip4/icmp4.hpp" -#include "dns/client.hpp" -#include "tcp/tcp.hpp" -#include "super_stack.hpp" - -namespace net { - - class DHClient; - - /** A complete IP4 network stack */ - class Inet4 : public Inet{ - public: - - using Vip4_list = Vip_list; - - std::string ifname() const override - { return nic_.device_name(); } - - MAC::Addr link_addr() const override - { return nic_.mac(); } - - hw::Nic& nic() override - { return nic_; } - - IP4::addr ip_addr() const override - { return ip4_addr_; } - - IP4::addr netmask() const override - { return netmask_; } - - IP4::addr gateway() const override - { return gateway_; } - - IP4::addr dns_addr() const override - { return dns_server_; } - - IP4::addr broadcast_addr() const override - { return ip4_addr_ | ( ~ netmask_); } - - IP4& ip_obj() override - { return ip4_; } - - void cache_link_addr(IP4::addr ip, MAC::Addr mac) override - { arp_.cache(ip, mac); } - - void flush_link_cache() override - { arp_.flush_cache(); } - - void set_link_cache_flush_interval(std::chrono::minutes min) override - { arp_.set_cache_flush_interval(min); } - - /** Get the TCP-object belonging to this stack */ - TCP& tcp() override { return tcp_; } - - /** Get the UDP-object belonging to this stack */ - UDP& udp() override { return udp_; } - - /** Get the ICMP-object belonging to this stack */ - ICMPv4& icmp() override { return icmp_; } - - /** Get the DHCP client (if any) */ - auto dhclient() { return dhcp_; } - - /** - * @brief Disable or enable Path MTU Discovery (enabled by default) - * RFC 1191 - * If enabled, it sets the Don't Fragment flag on each IP4 packet - * TCP and UDP acts based on this being enabled or not - * - * @param[in] on Enables Path MTU Discovery if true, disables if false - * @param[in] aged Number of minutes that indicate that a PMTU value - * has grown stale and should be reset/increased - * This could be set to "infinity" (PMTU should never be - * increased) by setting the value to IP4::INFINITY - */ - void set_path_mtu_discovery(bool on, uint16_t aged = 10) override - { ip4_.set_path_mtu_discovery(on, aged); } - - /** - * @brief Triggered by IP when a Path MTU value has grown stale and the value - * is reset (increased) to check if the PMTU for the path could have increased - * This is NOT a change in the Path MTU in response to receiving an ICMP Too Big message - * and no retransmission of packets should take place - * - * @param[in] dest The destination/path - * @param[in] pmtu The reset PMTU value - */ - void reset_pmtu(Socket dest, IP4::PMTU pmtu) override - { tcp_.reset_pmtu(dest, pmtu); /* Maybe later: udp_.reset_pmtu(dest, pmtu);*/ } - - /** - * Error reporting - * - * Including ICMP error report in accordance with RFC 1122 and handling of ICMP - * too big messages in accordance with RFC 1191, 1981 and 4821 (Path MTU Discovery - * and Packetization Layer Path MTU Discovery) - * - * Forwards errors to the transport layer (UDP and TCP) - */ - void error_report(Error& err, Packet_ptr orig_pckt) override; - - /** - * Set the forwarding delegate used by this stack. - * If set it will get all incoming packets not intended for this stack. - * NOTE: This delegate is expected to call the forward chain - */ - void set_forward_delg(Forward_delg fwd) override { - ip4_.set_packet_forwarding(fwd); - } - - /** - * Assign a delegate that checks if we have a route to a given IP - */ - void set_route_checker(Route_checker delg) override - { arp_.set_proxy_policy(delg); } - - /** - * Get the forwarding delegate used by this stack. - */ - Forward_delg forward_delg() override - { return ip4_.forward_delg(); } - - - Packet_ptr create_packet() override { - return nic_.create_packet(nic_.frame_offset_link()); - } - - /** - * Provision an IP packet - * @param proto : IANA protocol number. - */ - IP4::IP_packet_ptr create_ip_packet(Protocol proto) override { - auto raw = nic_.create_packet(nic_.frame_offset_link()); - auto ip_packet = static_unique_ptr_cast(std::move(raw)); - - ip_packet->init(proto); - - return ip_packet; - } - - IP_packet_factory ip_packet_factory() override - { return IP_packet_factory{this, &Inet4::create_ip_packet}; } - - /** MTU retreived from Nic on construction */ - uint16_t MTU() const override - { return MTU_; } - - /** - * @func a delegate that provides a hostname and its address, which is 0 if the - * name @hostname was not found. Note: Test with INADDR_ANY for a 0-address. - **/ - void resolve(const std::string& hostname, - resolve_func func, - bool force = false) override - { - dns_.resolve(this->dns_server_, hostname, func, force); - } - - void resolve(const std::string& hostname, - IP4::addr server, - resolve_func func, - bool force = false) override - { - dns_.resolve(server, hostname, func, force); - } - - void set_domain_name(std::string domain_name) override - { this->domain_name_ = std::move(domain_name); } - - const std::string& domain_name() const override - { return this->domain_name_; } - - void set_gateway(IP4::addr gateway) override - { - this->gateway_ = gateway; - } - - void set_dns_server(IP4::addr server) override - { - this->dns_server_ = server; - } - - /** - * @brief Try to negotiate DHCP - * @details Initialize DHClient if not present and tries to negotitate dhcp. - * Also takes an optional timeout parameter and optional timeout function. - * - * @param timeout number of seconds before request should timeout - * @param dhcp_timeout_func DHCP timeout handler - */ - void negotiate_dhcp(double timeout = 10.0, dhcp_timeout_func = nullptr) override; - - bool is_configured() const override - { - return ip4_addr_ != 0; - } - - // handler called after the network is configured, - // either by DHCP or static network configuration - void on_config(on_configured_func handler) override - { - configured_handlers_.push_back(handler); - } - - /** We don't want to copy or move an IP-stack. It's tied to a device. */ - Inet4(Inet4&) = delete; - Inet4(Inet4&&) = delete; - Inet4& operator=(Inet4) = delete; - Inet4 operator=(Inet4&&) = delete; - - void network_config(IP4::addr addr, - IP4::addr nmask, - IP4::addr gateway, - IP4::addr dns = IP4::ADDR_ANY) override; - - virtual void - reset_config() override - { - this->ip4_addr_ = IP4::ADDR_ANY; - this->gateway_ = IP4::ADDR_ANY; - this->netmask_ = IP4::ADDR_ANY; - } - - // register a callback for receiving signal on free packet-buffers - virtual void - on_transmit_queue_available(transmit_avail_delg del) override { - tqa.push_back(del); - } - - size_t transmit_queue_available() override { - return nic_.transmit_queue_available(); - } - - size_t buffers_available() override { - return nic_.buffers_available(); - } - size_t buffers_total() override { - return nic_.buffers_total(); - } - - void force_start_send_queues() override; - - void move_to_this_cpu() override; - - int get_cpu_id() const noexcept override { - return this->cpu_id; - } - - /** Return the stack on the given Nic */ - template - static auto&& stack() - { - return Super_stack::get(N); - } - - /** Static IP config */ - template - static auto&& ifconfig( - IP4::addr addr, - IP4::addr nmask, - IP4::addr gateway, - IP4::addr dns = IP4::ADDR_ANY) - { - stack().network_config(addr, nmask, gateway, dns); - return stack(); - } - - /** DHCP config */ - template - static auto& ifconfig(double timeout = 10.0, dhcp_timeout_func on_timeout = nullptr) - { - if (timeout > 0.0) - stack().negotiate_dhcp(timeout, on_timeout); - return stack(); - } - - /** Add virtual IP4 address as loopback */ - const Vip4_list virtual_ips() const noexcept override - { return vip4s_; } - - /** Check if IP4 address is virtual loopback */ - bool is_loopback(IP4::addr a) const override - { - return a.is_loopback() - or std::find( vip4s_.begin(), vip4s_.end(), a) != vip4s_.end(); - } - - /** Add IP4 address as virtual loopback */ - void add_vip(IP4::addr a) override - { - if (not is_loopback(a)) { - INFO("Inet4", "Adding virtual IP address %s", a.to_string().c_str()); - vip4s_.emplace_back(a); - } - } - - /** Add IP4 address as virtual loopback */ - void remove_vip(IP4::addr a) override - { - auto it = std::find(vip4s_.begin(), vip4s_.end(), a); - if (it != vip4s_.end()) - vip4s_.erase(it); - } - - IP4::addr get_source_addr(IP4::addr dest) override - { - - if (dest.is_loopback()) - return {127,0,0,1}; - - if (is_loopback(dest)) - return dest; - - return ip_addr(); - } - - bool is_valid_source(IP4::addr src) const override - { return src == ip_addr() or is_loopback(src); } - - virtual std::shared_ptr& conntrack() override - { return conntrack_; } - - virtual const std::shared_ptr& conntrack() const override - { return conntrack_; } - - virtual void enable_conntrack(std::shared_ptr ct) override; - - virtual Port_utils& tcp_ports() override - { return tcp_ports_; } - - virtual Port_utils& udp_ports() override - { return udp_ports_; } - - /** Initialize with ANY_ADDR */ - Inet4(hw::Nic& nic); - - private: - - void process_sendq(size_t); - // delegates registered to get signalled about free packets - std::vector tqa; - - IP4::addr ip4_addr_; - IP4::addr netmask_; - IP4::addr gateway_; - IP4::addr dns_server_; - - Vip4_list vip4s_ = {{127,0,0,1}}; - - // This is the actual stack - hw::Nic& nic_; - Arp arp_; - IP4 ip4_; - ICMPv4 icmp_; - UDP udp_; - TCP tcp_; - - Port_utils tcp_ports_; - Port_utils udp_ports_; - - std::shared_ptr conntrack_; - - // we need this to store the cache per-stack - DNSClient dns_; - std::string domain_name_; - - std::shared_ptr dhcp_{}; - - std::vector configured_handlers_; - - int cpu_id; - const uint16_t MTU_; - - friend class Super_stack; - }; -} - -#endif diff --git a/api/net/inet_common.hpp b/api/net/inet_common.hpp index de32e37496..01b89a87fd 100644 --- a/api/net/inet_common.hpp +++ b/api/net/inet_common.hpp @@ -84,13 +84,15 @@ namespace net { * http://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml */ enum class Protocol : uint8_t { - HOPOPT = 0, - ICMPv4 = 1, - IPv4 = 4, // IPv4 encapsulation - TCP = 6, - UDP = 17, - IPv6 = 41, // IPv6 encapsulation - ICMPv6 = 58 + HOPOPT = 0, + ICMPv4 = 1, + IPv4 = 4, // IPv4 encapsulation + TCP = 6, + UDP = 17, + IPv6 = 41, // IPv6 encapsulation + ICMPv6 = 58, + IPv6_NONXT = 59, + OPTSV6 = 60 }; /** diff --git a/api/net/ip4/ip4.hpp b/api/net/ip4/ip4.hpp index b954f8006a..332a53413d 100644 --- a/api/net/ip4/ip4.hpp +++ b/api/net/ip4/ip4.hpp @@ -18,12 +18,13 @@ #ifndef NET_IP4_IP4_HPP #define NET_IP4_IP4_HPP +#include #include "addr.hpp" #include "header.hpp" #include "packet_ip4.hpp" #include #include -#include +#include #include #include @@ -31,6 +32,8 @@ namespace net { + class Inet; + /** IP4 layer */ class IP4 { public: @@ -55,14 +58,18 @@ namespace net { SIX = 2002 }; - using Stack = Inet; + using Stack = class Inet; using addr = ip4::Addr; using header = ip4::Header; using IP_packet = PacketIP4; using IP_packet_ptr = std::unique_ptr; + using IP_packet_factory = delegate; using downstream_arp = delegate; using drop_handler = delegate; + using Forward_delg = delegate; using PMTU = uint16_t; + using Port_utils = std::map; + using resolve_func = delegate; /** Initialize. Sets a dummy linklayer out. */ explicit IP4(Stack&) noexcept; @@ -86,16 +93,14 @@ namespace net { /* Maximum Datagram Data Size */ - uint16_t MDDS() const - { return stack_.MTU() - sizeof(ip4::Header); } + uint16_t MDDS() const; /** * @brief Get the default MTU for the OS, including the size of the ip4::Header * * @return The default Path MTU value */ - uint16_t default_PMTU() const noexcept - { return stack_.MTU(); } + uint16_t default_PMTU() const noexcept; /** Upstream: Input from link layer */ void receive(Packet_ptr, const bool link_bcast); @@ -122,7 +127,7 @@ namespace net { { drop_handler_ = s; } /** Set handler for packets not addressed to this interface (upstream) */ - void set_packet_forwarding(Stack::Forward_delg fwd) + void set_packet_forwarding(Forward_delg fwd) { forward_packet_ = fwd; } /** Set linklayer out (downstream) */ @@ -142,7 +147,7 @@ namespace net { upstream tcp_handler() { return tcp_handler_; } - Stack::Forward_delg forward_delg() + Forward_delg forward_delg() { return forward_packet_; } downstream_arp linklayer_out() @@ -167,9 +172,7 @@ namespace net { * * Returns the IPv4 address associated with this interface **/ - const addr local_ip() const { - return stack_.ip_addr(); - } + const addr local_ip() const; /** * @brief Determines if the packet is for me (this host). @@ -468,7 +471,7 @@ namespace net { upstream tcp_handler_ = nullptr; /** Packet forwarding */ - Stack::Forward_delg forward_packet_; + Forward_delg forward_packet_; // Filter chains Filter_chain prerouting_chain_{"Prerouting", {}}; diff --git a/api/net/ip4/packet_icmp4.hpp b/api/net/ip4/packet_icmp4.hpp index 790fb790bd..437b33c369 100644 --- a/api/net/ip4/packet_icmp4.hpp +++ b/api/net/ip4/packet_icmp4.hpp @@ -125,7 +125,7 @@ namespace icmp4 { { } /** Provision fresh packet from factory **/ - Packet(Inet::IP_packet_factory create) + Packet(IP4::IP_packet_factory create) : pckt_{create(Protocol::ICMPv4)} { pckt_->increment_data_end(sizeof(Header)); diff --git a/api/net/ip4/udp.hpp b/api/net/ip4/udp.hpp index a52dc2233e..a2cb22404e 100644 --- a/api/net/ip4/udp.hpp +++ b/api/net/ip4/udp.hpp @@ -23,7 +23,6 @@ #include #include -#include "../inet.hpp" #include "ip4.hpp" #include #include @@ -32,6 +31,7 @@ namespace net { + class Inet; class PacketUDP; class UDPSocket; @@ -48,6 +48,7 @@ namespace net { using Packet_ptr = std::unique_ptr>; using Stack = IP4::Stack; + using Port_utils = IP4::Port_utils; using Sockets = std::map; @@ -99,8 +100,7 @@ namespace net { //////////////////////////////////////////// - addr_t local_ip() const - { return stack_.ip_addr(); } + addr_t local_ip() const; /** Input from network layer */ void receive(net::Packet_ptr); @@ -124,21 +124,16 @@ namespace net { void transmit(UDP::Packet_ptr udp); //! @param port local port - UDPSocket& bind(port_t port) - { return bind({stack_.ip_addr(), port}); } + UDPSocket& bind(port_t port); UDPSocket& bind(const Socket socket); //! returns a new UDP socket bound to a random port - UDPSocket& bind() - { return bind(stack_.ip_addr()); } - + UDPSocket& bind(); UDPSocket& bind(const ip4::Addr addr); bool is_bound(const Socket) const; - - bool is_bound(const port_t port) const - { return is_bound({stack_.ip_addr(), port}); } + bool is_bound(const port_t port) const; /** Close a socket **/ void close(const Socket socket); @@ -158,8 +153,7 @@ namespace net { // create and transmit @num packets from sendq void process_sendq(size_t num); - uint16_t max_datagram_size() noexcept - { return stack().ip_obj().MDDS() - sizeof(header); } + uint16_t max_datagram_size() noexcept; class Port_in_use_exception : public UDP_error { public: @@ -181,7 +175,7 @@ namespace net { downstream network_layer_out_; Stack& stack_; Sockets sockets_; - Stack::Port_utils& ports_; + Port_utils& ports_; // the async send queue std::deque sendq; diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp new file mode 100644 index 0000000000..8bca66a7d8 --- /dev/null +++ b/api/net/ip6/addr.hpp @@ -0,0 +1,209 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef NET_IP6_ADDR_HPP +#define NET_IP6_ADDR_HPP + +#include +#include +#include +#include + +namespace net { +namespace ip6 { + +/** + * This type is thrown when creating an instance of Addr + * with a std::string that doesn't represent a valid IPv6 + * Address + */ +struct Invalid_Address : public std::runtime_error { + using runtime_error::runtime_error; +}; //< struct Invalid_Address + +/** + * IPv6 Address representation + */ +struct Addr { + Addr() + : i32{0, 0, 0, 0} {} + + Addr(uint16_t a1, uint16_t a2, uint16_t b1, uint16_t b2, + uint16_t c1, uint16_t c2, uint16_t d1, uint16_t d2) + { + i16[0] = htons(a1); i16[1] = htons(a2); + i16[2] = htons(b1); i16[3] = htons(b2); + i16[4] = htons(c1); i16[5] = htons(c2); + i16[6] = htons(d1); i16[7] = htons(d2); + } + + Addr(uint32_t a, uint32_t b, uint32_t c, uint32_t d) + { + i32[0] = htonl(a); i32[1] = htonl(b); + i32[2] = htonl(c); i32[3] = htonl(d); + } + + Addr(const Addr& a) + { + for (int i = 0; i < 4; i++) { + i32[i] = a.i32[i]; + } + } + + // returns this IPv6 Address as a string + std::string str() const { + char ipv6_addr[32]; + snprintf(ipv6_addr, sizeof(ipv6_addr), + "%0x:%0x:%0x:%0x:%0x:%0x:%0x:%0x", + ntohs(i16[0]), ntohs(i16[1]), ntohs(i16[2]), + ntohs(i16[3]), ntohs(i16[4]), ntohs(i16[5]), + ntohs(i16[6]), ntohs(i16[7])); + return ipv6_addr; + } + + /** + * Get a string representation of this type + * + * @return A string representation of this type + */ + std::string to_string() const + { return str(); } + + // multicast IPv6 Addresses + static const Addr node_all_nodes; // RFC 4921 + static const Addr node_all_routers; // RFC 4921 + static const Addr node_mDNSv6; // RFC 6762 (multicast DNSv6) + + // unspecified link-local Address + static const Addr link_unspecified; + + // RFC 4291 2.4.6: + // Link-Local Addresses are designed to be used for Addressing on a + // single link for purposes such as automatic Address configuration, + // neighbor discovery, or when no routers are present. + static const Addr link_all_nodes; // RFC 4921 + static const Addr link_all_routers; // RFC 4921 + static const Addr link_mDNSv6; // RFC 6762 + + static const Addr link_dhcp_servers; // RFC 3315 + static const Addr site_dhcp_servers; // RFC 3315 + + // returns true if this Addr is a IPv6 multicast Address + bool is_multicast() const + { + /** + RFC 4291 2.7 Multicast Addresses + + An IPv6 multicast Address is an identifier for a group of interfaces + (typically on different nodes). An interface may belong to any + number of multicast groups. Multicast Addresses have the following format: + | 8 | 4 | 4 | 112 bits | + +------ -+----+----+---------------------------------------------+ + |11111111|flgs|scop| group ID | + +--------+----+----+---------------------------------------------+ + **/ + return i8[0] == 0xFF; + } + + /** + * + **/ + bool is_loopback() const noexcept + { return i32[0] == 1 && i32[1] == 0 && + i32[2] == 0 && i32[3] == 0; } + + /** + * Assignment operator + */ + Addr& operator=(const Addr other) noexcept { + i32[0] = other.i32[0]; + i32[1] = other.i32[1]; + i32[2] = other.i32[2]; + i32[3] = other.i32[3]; + return *this; + } + + /** + * Operator to check for equality + */ + bool operator==(const Addr other) const noexcept + { return i32[0] == other.i32[0] && i32[1] == other.i32[1] && + i32[2] == other.i32[2] && i32[3] == other.i32[3]; } + + /** + * Operator to check for inequality + */ + bool operator!=(const Addr other) const noexcept + { return not (*this == other); } + + /** + * Operator to check for less-than relationship + */ + bool operator<(const Addr other) const noexcept + { return i32[3] < other.i32[3] && + i32[2] < other.i32[2] && + i32[1] < other.i32[1] && + i32[0] < other.i32[0]; } + + /** + * Operator to check for less-than-or-equal relationship + */ + bool operator<=(const Addr other) const noexcept + { return (*this < other or *this == other); } + + /** + * Operator to check for greater-than relationship + */ + bool operator>(const Addr other) const noexcept + { return not (*this <= other); } + + /** + * Operator to check for greater-than-or-equal relationship + */ + bool operator>=(const Addr other) const noexcept + { return (*this > other or *this == other); } + + /** + * Operator to perform a bitwise-and operation on the given + * IPv4 addresses + */ + Addr operator&(const Addr other) const noexcept + { return Addr{i32[3] & other.i32[3], + i32[2] & other.i32[2], + i32[1] & other.i32[1], + i32[0] & other.i32[0] }; } + + Addr operator|(const Addr other) const noexcept + { return Addr{i32[3] | other.i32[3], + i32[2] | other.i32[2], + i32[1] | other.i32[1], + i32[0] | other.i32[0] }; } + + Addr operator~() const noexcept + { return Addr{~i32[3], ~i32[2], ~i32[1], ~i32[0]}; } + + union + { + uint32_t i32[4]; + uint16_t i16[8]; + uint8_t i8[16]; + }; +} __attribute__((packed)); //< struct Addr +} //< namespace ip6 +} //< namespace net +#endif diff --git a/api/net/ip6/header.hpp b/api/net/ip6/header.hpp new file mode 100644 index 0000000000..8e8e198cd2 --- /dev/null +++ b/api/net/ip6/header.hpp @@ -0,0 +1,67 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef NET_IP6_HEADER_HPP +#define NET_IP6_HEADER_HPP + +#include +#include +#define IP6_HEADER_LEN 40 + +namespace net { +namespace ip6 { + +/** + * This type is used to represent the standard IPv6 header + */ +struct Header { + uint32_t version:4, + traffic_class:8, + flow_label:20; + uint16_t payload_length; + uint8_t next_header; + uint8_t hop_limit; + Addr saddr; + Addr daddr; +}; //< struct Header + + +struct ExtensionHeader +{ + uint8_t next_header; + uint8_t hdr_ext_len; + uint16_t opt_1; + uint32_t opt_2; + + Protocol next() const + { + return static_cast(next_header); + } + uint8_t size() const + { + return sizeof(ExtensionHeader) + hdr_ext_len; + } + uint8_t extended() const + { + return hdr_ext_len; + } +}; + +} //< namespace ip6 +} //< namespace net +#endif //< NET_IP6_HEADER_HPP diff --git a/api/net/ip6/icmp6.hpp b/api/net/ip6/icmp6.hpp index 6e6822e644..858b9122c2 100644 --- a/api/net/ip6/icmp6.hpp +++ b/api/net/ip6/icmp6.hpp @@ -17,8 +17,12 @@ #pragma once -#include "../util.hpp" -#include "packet_ip6.hpp" +#ifndef NET_IP6_ICMPv6_HPP +#define NET_IP6_ICMPv6_HPP + +//#include "packet_icmp6.hpp" +#include +#include namespace net { @@ -26,16 +30,66 @@ namespace net class ICMPv6 { +#if 0 + using ICMP_type = ICMP_error::ICMP_type; + using ICMP_code = ICMP_error::ICMP_code; + public: - static const int ECHO_REQUEST = 128; - static const int ECHO_REPLY = 129; + using Stack = IP6::Stack; + using Tuple = std::pair; // identifier and sequence number + using icmp_func = delegate; + + static const int SEC_WAIT_FOR_REPLY = 40; + + // Initialize + ICMPv6(Stack&); + + // Input from network layer + void receive(Packet_ptr); + + // Delegate output to network layer + inline void set_network_out(downstream s) + { network_layer_out_ = s; }; + + /** + * Destination Unreachable sent from host because of port (UDP) or protocol (IP6) unreachable + */ + void destination_unreachable(Packet_ptr pckt, icmp6::code::Dest_unreachable code); + + /** + * + */ + void redirect(Packet_ptr pckt, icmp6::code::Redirect code); - static const int ND_ROUTER_SOL = 133; - static const int ND_ROUTER_ADV = 134; - static const int ND_NEIGHB_SOL = 135; - static const int ND_NEIGHB_ADV = 136; - static const int ND_REDIRECT = 137; + /** + * Sending a Time Exceeded message from a host when fragment reassembly time exceeded (code 1) + * Sending a Time Exceeded message from a gateway when time to live exceeded in transit (code 0) + */ + void time_exceeded(Packet_ptr pckt, icmp6::code::Time_exceeded code); + /** + * Sending a Parameter Problem message if the gateway or host processing a datagram finds a problem with + * the header parameters such that it cannot complete processing the datagram. The message is only sent if + * the error caused the datagram to be discarded + * Code 0 means Pointer (uint8_t after checksum) indicates the error/identifies the octet where an error was detected + * in the IP header + * Code 1 means that a required option is missing + */ + void parameter_problem(Packet_ptr pckt, uint8_t error_pointer); + + // May + void timestamp_request(IP6::addr ip); + void timestamp_reply(icmp6::Packet& req); + + void ping(IP6::addr ip); + void ping(IP6::addr ip, icmp_func callback, int sec_wait = SEC_WAIT_FOR_REPLY); + + void ping(const std::string& hostname); + void ping(const std::string& hostname, icmp_func callback, int sec_wait = SEC_WAIT_FOR_REPLY); +#endif + + +#if 0 typedef uint8_t type_t; typedef int (*handler_t)(ICMPv6&, std::shared_ptr&); @@ -143,7 +197,7 @@ namespace net // new total packet length set_size(sizeof(IP6::full_header) + icmp_len); } - +#endif }; - } +#endif diff --git a/api/net/ip6/ip6.hpp b/api/net/ip6/ip6.hpp index 9c2f5eab9e..083199fdbf 100644 --- a/api/net/ip6/ip6.hpp +++ b/api/net/ip6/ip6.hpp @@ -18,292 +18,204 @@ #ifndef NET_IP6_IP6_HPP #define NET_IP6_IP6_HPP -#include -#include -#include "../packet.hpp" -#include "../util.hpp" +#include "addr.hpp" +#include "header.hpp" +#include "packet_ip6.hpp" -#include -#include -#include -#include -#include +#include +#include +#include namespace net { - class PacketIP6; - /** IP6 layer skeleton */ + class Inet; + + /** IP6 layer */ class IP6 { public: - /** Known transport layer protocols. */ - enum proto - { - PROTO_HOPOPT = 0, // IPv6 hop-by-hop - - PROTO_ICMPv4 = 1, - PROTO_TCP = 6, - PROTO_UDP = 17, - - PROTO_ICMPv6 = 58, // IPv6 ICMP - PROTO_NoNext = 59, // no next-header - PROTO_OPTSv6 = 60, // dest options - }; - - struct addr - { - // constructors - addr() - : i32{0, 0, 0, 0} {} - addr(uint16_t a1, uint16_t a2, uint16_t b1, uint16_t b2, - uint16_t c1, uint16_t c2, uint16_t d1, uint16_t d2) - { - i16[0] = a1; i16[1] = a2; - i16[2] = b1; i16[3] = b2; - i16[4] = c1; i16[5] = c2; - i16[6] = d1; i16[7] = d2; - /*i128 = _mm_set_epi16( - htons(d2), htons(d1), - htons(c2), htons(c1), - htons(b2), htons(b1), - htons(a2), htons(a1));*/ - } - addr(uint32_t a, uint32_t b, uint32_t c, uint32_t d) - { - i32[0] = a; i32[1] = b; i32[2] = c; i32[3] = d; - //i128 = _mm_set_epi32(d, c, b, a); - } - addr(const addr& a) - { - for (int i = 0; i < 4; i++) - i32[i] = a.i32[i]; - } - // move constructor - addr& operator= (const addr& a) - { - for (int i = 0; i < 4; i++) - i32[i] = a.i32[i]; - //i128 = a.i128; - return *this; - } - - // comparison functions - bool operator== (const addr& a) const - { - // i128 == a.i128: - for (int i = 0; i < 4; i++) - if (i32[i] != a.i32[i]) return false; - return true; - //__m128i cmp = _mm_cmpeq_epi32(i128, a.i128); - //return _mm_cvtsi128_si32(cmp); - } - bool operator!= (const addr& a) const - { - return !this->operator==(a); - } - - // returns this IPv6 address as a string - std::string str() const; - - // multicast IPv6 addresses - static const addr node_all_nodes; // RFC 4921 - static const addr node_all_routers; // RFC 4921 - static const addr node_mDNSv6; // RFC 6762 (multicast DNSv6) - - // unspecified link-local address - static const addr link_unspecified; - - // RFC 4291 2.4.6: - // Link-Local addresses are designed to be used for addressing on a - // single link for purposes such as automatic address configuration, - // neighbor discovery, or when no routers are present. - static const addr link_all_nodes; // RFC 4921 - static const addr link_all_routers; // RFC 4921 - static const addr link_mDNSv6; // RFC 6762 - - static const addr link_dhcp_servers; // RFC 3315 - static const addr site_dhcp_servers; // RFC 3315 - - // returns true if this addr is a IPv6 multicast address - bool is_multicast() const - { - /** - RFC 4291 2.7 Multicast Addresses - - An IPv6 multicast address is an identifier for a group of interfaces - (typically on different nodes). An interface may belong to any - number of multicast groups. Multicast addresses have the following format: - | 8 | 4 | 4 | 112 bits | - +------ -+----+----+---------------------------------------------+ - |11111111|flgs|scop| group ID | - +--------+----+----+---------------------------------------------+ - **/ - return i8[0] == 0xFF; - } - - union - { - //__m128i i128; - uint32_t i32[ 4]; - uint16_t i16[ 8]; - uint8_t i8[16]; - }; - }; - -#pragma pack(push, 1) - class header - { - public: - uint8_t version() const - { - return (scanline[0] & 0xF0) >> 4; - } - uint8_t tclass() const - { - return ((scanline[0] & 0xF000) >> 12) + - (scanline[0] & 0xF); - } - // initializes the first scanline with the IPv6 version - void init_scan0() - { - scanline[0] = 6u >> 4; - } - - uint16_t size() const - { - return ((scanline[1] & 0x00FF) << 8) + - ((scanline[1] & 0xFF00) >> 8); - } - void set_size(uint16_t newsize) - { - scanline[1] &= 0xFFFF0000; - scanline[1] |= htons(newsize); - } - - uint8_t next() const - { - return (scanline[1] >> 16) & 0xFF; - } - void set_next(uint8_t next) - { - scanline[1] &= 0xFF00FFFF; - scanline[1] |= next << 16; - } - uint8_t hoplimit() const - { - return (scanline[1] >> 24) & 0xFF; - } - void set_hoplimit(uint8_t limit = 64) - { - scanline[1] &= 0x00FFFFFF; - scanline[1] |= limit << 24; - } - - private: - uint32_t scanline[2]; - public: - addr src; - addr dst; - }; - - struct options_header - { - uint8_t next_header; - uint8_t hdr_ext_len; - uint16_t opt_1; - uint32_t opt_2; - - uint8_t next() const - { - return next_header; - } - uint8_t size() const - { - return sizeof(options_header) + hdr_ext_len; - } - uint8_t extended() const - { - return hdr_ext_len; - } - }; -#pragma pack(pop) - - struct full_header - { - Ethernet::header eth_hdr; - IP6::header ip6_hdr; - }; - - // downstream delegate for transmit() - typedef delegate&)> downstream6; - typedef downstream6 upstream6; - - /** Constructor. Requires ethernet to latch on to. */ - IP6(const addr& local); - - const IP6::addr& local_ip() const - { - return local; - } - - uint8_t parse6(uint8_t*& reader, uint8_t next); - - static std::string protocol_name(uint8_t protocol) - { - switch (protocol) - { - case PROTO_HOPOPT: - return "IPv6 Hop-By-Hop (0)"; - - case PROTO_TCP: - return "TCPv6 (6)"; - case PROTO_UDP: - return "UDPv6 (17)"; - - case PROTO_ICMPv6: - return "ICMPv6 (58)"; - case PROTO_NoNext: - return "No next header (59)"; - case PROTO_OPTSv6: - return "IPv6 destination options (60)"; - - default: - return "Unknown: " + std::to_string(protocol); - } - } - - // handler for upstream IPv6 packets - void bottom(Packet_ptr pckt); - - // transmit packets to the ether - void transmit(std::shared_ptr& pckt); - - // modify upstream handlers - inline void set_handler(uint8_t proto, upstream& handler) - { - proto_handlers[proto] = handler; - } - - inline void set_linklayer_out(downstream func) - { - _linklayer_out = func; - } - - // creates a new IPv6 packet to be sent over the ether - static std::shared_ptr create(uint8_t proto, - Ethernet::addr ether_dest, const IP6::addr& dest); + enum class Drop_reason + { None, Bad_source, Bad_destination, Wrong_version, + Unknown_proto }; + + enum class Direction + { Upstream, Downstream }; + + using Stack = class Inet; + using addr = ip6::Addr; + using header = ip6::Header; + using IP_packet = PacketIP6; + using IP_packet_ptr = std::unique_ptr; + using downstream_ndp = delegate; + using drop_handler = delegate; + using Forward_delg = delegate; + using PMTU = uint16_t; + + static const addr ADDR_ANY; + static const addr ADDR_LOOPBACK; + + /** Initialize. Sets a dummy linklayer out. */ + explicit IP6(Stack&) noexcept; + + // + // Delegate setters + // + + /** Set ICMP protocol handler (upstream)*/ + void set_icmp_handler(upstream s) + { icmp_handler_ = s; } + + /** Set UDP protocol handler (upstream)*/ + void set_udp_handler(upstream s) + { udp_handler_ = s; } + + /** Set TCP protocol handler (upstream) */ + void set_tcp_handler(upstream s) + { tcp_handler_ = s; } + + /** Set packet dropped handler */ + void set_drop_handler(drop_handler s) + { drop_handler_ = s; } + + /** Set handler for packets not addressed to this interface (upstream) */ + void set_packet_forwarding(Forward_delg fwd) + { forward_packet_ = fwd; } + + /** Set linklayer out (downstream) */ + void set_ndp_out(downstream_ndp s) + { ndp_out_ = s; } + + void set_linklayer_out(downstream_link link) + { linklayer_out_ = link; } + + /** Upstream: Input from link layer */ + void receive(Packet_ptr, const bool link_bcast); + + + // + // Delegate getters + // + + upstream icmp_handler() + { return icmp_handler_; } + + upstream udp_handler() + { return udp_handler_; } + + upstream tcp_handler() + { return tcp_handler_; } + + Forward_delg forward_delg() + { return forward_packet_; } + + /** + * Downstream: Receive data from above and transmit + * + * @note: The following *must be set* in the packet: + * + * * Destination IP + * * Protocol + * + * Source IP *can* be set - if it's not, IP4 will set it + */ + void transmit(Packet_ptr); + void ship(Packet_ptr, addr next_hop = IP6::ADDR_ANY, Conntrack::Entry_ptr ct = nullptr); + + /** + * \brief + * + * Returns the IPv4 address associated with this interface + **/ + const addr local_ip() const; + + /** + * @brief Determines if the packet is for me (this host). + * + * @param[in] dst The destination + * + * @return True if for me, False otherwise. + */ + bool is_for_me(ip6::Addr dst) const; + void ipv6_ext_header_receive(Packet_ptr pckt); + + /// + /// PACKET FILTERING + /// + + /** + * Packet filtering hooks for firewall, NAT, connection tracking etc. + **/ + + /** Packets pass through prerouting chain before routing decision */ + Filter_chain& prerouting_chain() + { return prerouting_chain_; } + + /** Packets pass through postrouting chain after routing decision */ + Filter_chain& postrouting_chain() + { return postrouting_chain_; } + + /** Packets pass through input chain before hitting protocol handlers */ + Filter_chain& input_chain() + { return input_chain_; } + + /** Packets pass through output chain after exiting protocol handlers */ + Filter_chain& output_chain() + { return output_chain_; } + + + /** + * Stats getters + **/ + uint64_t get_packets_rx() + { return packets_rx_; } + + uint64_t get_packets_tx() + { return packets_tx_; } + + uint64_t get_packets_dropped() + { return packets_dropped_; } + + /** Drop incoming packets invalid according to RFC */ + IP_packet_ptr drop_invalid_in(IP_packet_ptr packet); + + /** Drop outgoing packets invalid according to RFC */ + IP_packet_ptr drop_invalid_out(IP_packet_ptr packet); private: - addr local; + /** Stats */ + uint64_t& packets_rx_; + uint64_t& packets_tx_; + uint32_t& packets_dropped_; + Stack& stack_; - /** Downstream: Linklayer output delegate */ - downstream _linklayer_out; /** Upstream delegates */ - std::map proto_handlers; - }; + upstream icmp_handler_ = nullptr; + upstream udp_handler_ = nullptr; + upstream tcp_handler_ = nullptr; + + /** Downstream delegates */ + downstream_ndp ndp_out_ = nullptr; + downstream_link linklayer_out_ = nullptr; + + /** Packet forwarding */ + Forward_delg forward_packet_; + + // Filter chains + Filter_chain prerouting_chain_{"Prerouting", {}}; + Filter_chain postrouting_chain_{"Postrouting", {}}; + Filter_chain input_chain_{"Input", {}}; + Filter_chain output_chain_{"Output", {}}; + + /** All dropped packets go here */ + drop_handler drop_handler_; + + /** Drop a packet, calling drop handler if set */ + IP_packet_ptr drop(IP_packet_ptr ptr, Direction direction, Drop_reason reason); + + }; //< class IP6 -} // namespace net +} //< namespace net #endif diff --git a/api/net/ip6/packet_ip6.hpp b/api/net/ip6/packet_ip6.hpp index f62b801123..10bea8134b 100644 --- a/api/net/ip6/packet_ip6.hpp +++ b/api/net/ip6/packet_ip6.hpp @@ -16,64 +16,170 @@ // limitations under the License. #pragma once -#include "ip6.hpp" +#ifndef IP6_PACKET_IP6_HPP +#define IP6_PACKET_IP6_HPP + +#include "header.hpp" +#include +#include namespace net { - - class PacketIP6 : public Packet - { + + /** IPv6 packet. */ + class PacketIP6 : public Packet { public: - IP6::header& ip6_header() - { - return ((IP6::full_header*) buffer())->ip6_hdr; - } - const IP6::header& ip6_header() const - { - return ((IP6::full_header*) buffer())->ip6_hdr; - } - Ethernet::header& eth_header() - { - return *(Ethernet::header*) buffer(); - } - - const IP6::addr& src() const + using Span = gsl::span; + using Cspan = gsl::span; + + // IPv6 header getters + + /** Get IP protocol version field. Must be 6 */ + ip6::Header& ip6_header() noexcept + { return *reinterpret_cast(layer_begin()); } + + uint8_t ip6_version() const noexcept + { return ip6_header().version; } + + bool is_ipv6() const noexcept + { return (ip6_header().version) == 6; } + + /** Get traffic class */ + uint8_t traffic_class() const noexcept + { return (ip6_header().traffic_class); } + + /** Get Differentiated Services Code Point (DSCP)*/ + DSCP ip_dscp() const noexcept + { return static_cast(traffic_class() >> 2); } + + /** Get Explicit Congestion Notification (ECN) bits */ + ECN ip_ecn() const noexcept + { return ECN(traffic_class() & 0x3); } + + /** Get flow label */ + uint32_t flow_label() const noexcept + { return (ip6_header().flow_label); } + + /** Get payload length */ + uint16_t payload_length() const noexcept + { return ntohs(ip6_header().payload_length); } + + /** Get next header */ + Protocol next_protocol() const noexcept + { return static_cast(ip6_header().next_header); } + + /** Get next header */ + uint8_t next_header() const noexcept + { return ip6_header().next_header; } + + /** Get hop limit */ + uint8_t hop_limit() const noexcept + { return ip6_header().hop_limit; } + + /** Get source address */ + const ip6::Addr& ip_src() const noexcept + { return ip6_header().saddr; } + + /** Get destination address */ + const ip6::Addr& ip_dst() const noexcept + { return ip6_header().daddr; } + + /** Get IP data length. */ + uint16_t ip_data_length() const noexcept { - return ip6_header().src; + Expects(size() and static_cast(size()) >= sizeof(ip6::Header)); + return size() - IP6_HEADER_LEN; } - void set_src(const IP6::addr& src) + + /** Get total data capacity of IP packet in bytes */ + uint16_t ip_capacity() const noexcept + { return capacity() - IP6_HEADER_LEN; } + + // IPv6 setters + // + /** Set IP version header field */ + void set_ip_version(uint8_t ver) noexcept { - ip6_header().src = src; + Expects(ver < 0x10); + ip6_header().version = ver; } - - const IP6::addr& dst() const - { - return ip6_header().dst; + + /** Set DSCP header bits */ + void set_ip_dscp(DSCP dscp) noexcept + { ip6_header().traffic_class |= (static_cast(dscp) << 2); } + + /** Set ECN header bits */ + void set_ip_ecn(ECN ecn) noexcept + { ip6_header().traffic_class |= (static_cast(ecn) & 0x3); } + + /** Set payload length */ + void set_ip_payload_length(uint16_t len) noexcept + { ip6_header().payload_length = htons(len); } + + /** Set next header */ + void set_ip_next_header(uint8_t next_header) noexcept + { ip6_header().next_header = next_header; } + + /** Set hop limit */ + void set_ip_hop_limit(uint8_t hop_limit) noexcept + { ip6_header().hop_limit = hop_limit; } + + /** Set source address header field */ + void set_ip_src(const ip6::Addr& addr) noexcept + { ip6_header().saddr = addr; } + + /** Set destination address header field */ + void set_ip_dst(const ip6::Addr& addr) noexcept + { ip6_header().daddr = addr; } + + /** Last modifications before transmission */ + void make_flight_ready() noexcept { + assert(ip6_header().next_header); + set_segment_length(); } - void set_dst(const IP6::addr& dst) - { - ip6_header().dst = dst; + + void init(Protocol proto = Protocol::HOPOPT) noexcept { + Expects(size() == 0); + auto& hdr = ip6_header(); + std::memset(&ip6_header(), 0, IP6_HEADER_LEN); + hdr.version = 6; + hdr.next_header = static_cast(proto); + hdr.payload_length = 0x1400; // Big-endian 20 + increment_data_end(IP6_HEADER_LEN); } - - uint8_t hoplimit() const - { - return ip6_header().hoplimit(); + + Span ip_data() { + return {ip_data_ptr(), ip_data_length()}; } - void set_hoplimit(uint8_t limit) - { - ip6_header().set_hoplimit(limit); + + Cspan ip_data() const { + return {ip_data_ptr(), ip_data_length()}; } - - // returns the protocol type of the next header - uint8_t next() const + + protected: + + /** Get pointer to IP data */ + Byte* ip_data_ptr() noexcept __attribute__((assume_aligned(4))) { - return ip6_header().next(); + return layer_begin() + IP6_HEADER_LEN; } - void set_next(uint8_t next) + + const Byte* ip_data_ptr() const noexcept __attribute__((assume_aligned(4))) { - ip6_header().set_next(next); + return layer_begin() + IP6_HEADER_LEN; } - - }; // PacketIP6 - -} + + private: + + /** + * Set IP6 payload length + */ + void set_segment_length() noexcept + { ip6_header().payload_length = htons(size()) - IP6_HEADER_LEN; } + + const ip6::Header& ip6_header() const noexcept + { return *reinterpret_cast(layer_begin()); } + + }; //< class PacketIP6 +} //< namespace net +#endif diff --git a/api/net/ip6/udp6.hpp b/api/net/ip6/udp6.hpp index f57bd11c3f..c5f7b5e904 100644 --- a/api/net/ip6/udp6.hpp +++ b/api/net/ip6/udp6.hpp @@ -15,136 +15,39 @@ // See the License for the specific language governing permissions and // limitations under the License. -#pragma once +#ifndef NET_IP6_UDP_HPP +#define NET_IP6_UDP_HPP -#include "../util.hpp" -#include "packet_ip6.hpp" +#include #include +#include +#include + +#include "../inet.hpp" +#include "ip6.hpp" +#include +#include +#include +#include namespace net { class PacketUDP6; + class UDPSocket; class UDPv6 { public: - typedef uint16_t port_t; - //typedef int (*listener_t)(std::shared_ptr& pckt); - typedef delegate& pckt)> listener_t; - - struct header - { - uint16_t src_port; - uint16_t dst_port; - uint16_t length; - uint16_t chksum; - } __attribute__((packed)); - - struct pseudo_header - { - IP6::addr src; - IP6::addr dst; - uint8_t zero; - uint8_t protocol; - uint16_t length; - } __attribute__((packed)); - - UDPv6(IP6::addr& local_ip) - : localIP(local_ip) {} - - // set the downstream delegate - inline void set_ip6_out(IP6::downstream6 del) - { - this->ip6_out = del; - } - - // packet from IP6 layer - int bottom(Packet_ptr pckt); - - // packet back TO IP6 layer for transmission - int transmit(std::shared_ptr& pckt); - - // register a listener function on @port - void listen(port_t port, listener_t func) - { - listeners[port] = func; - } - - // creates a new packet to be sent over the ether - std::shared_ptr create( - Ethernet::addr ether_dest, const IP6::addr& dest, port_t port); - - private: - std::map listeners; - // connection to IP6 layer - IP6::downstream6 ip6_out; - // this network stacks IPv6 address - IP6::addr& localIP; - }; - - class PacketUDP6 : public PacketIP6 - { - public: - UDPv6::header& header() - { - return *(UDPv6::header*) payload(); - } - const UDPv6::header& cheader() const - { - return *(UDPv6::header*) payload(); - } - - UDPv6::port_t src_port() const - { - return htons(cheader().src_port); - } - void set_src_port(UDPv6::port_t port) - { - header().src_port = htons(port); - } - UDPv6::port_t dst_port() const - { - return htons(cheader().dst_port); - } - void set_dst_port(UDPv6::port_t port) - { - header().dst_port = htons(port); - } - uint16_t length() const - { - return htons(cheader().length); - } - uint16_t checksum() const - { - return htons(cheader().chksum); - } + using addr_t = IP6::addr; + using port_t = uint16_t; - void set_length(uint16_t newlen) - { - // new total UDPv6 payload length - header().length = htons(sizeof(UDPv6::header) + newlen); - // new total IPv6 payload length - ip6_header().set_size( - sizeof(IP6::header) + sizeof(UDPv6::header) + newlen); - // new total packet length - set_size(sizeof(IP6::full_header) + sizeof(UDPv6::header) + newlen); - } + using Packet_ptr = std::unique_ptr>; + using Stack = IP6::Stack; - // generates a new checksum onto this UDPv6 packet - // returns the new generated checksum after modifying packet - uint16_t gen_checksum(); + using Sockets = std::map; - uint16_t data_length() const - { - return length() - sizeof(UDPv6::header); - } - char* data() - { - return (char*) payload() + sizeof(UDPv6::header); - } - const char* data() const - { - return (const char*) payload() + sizeof(UDPv6::header); - } + typedef delegate sendto_handler; + typedef delegate error_handler; }; } +#endif diff --git a/api/net/nat/napt.hpp b/api/net/nat/napt.hpp index 7dbbfcdb9a..00074e7ce1 100644 --- a/api/net/nat/napt.hpp +++ b/api/net/nat/napt.hpp @@ -22,7 +22,6 @@ #include #include #include -#include #include namespace net { @@ -33,7 +32,7 @@ namespace nat { */ class NAPT { public: - using Stack = Inet; + using Stack = Inet; public: diff --git a/api/net/netfilter.hpp b/api/net/netfilter.hpp index 5727473f8e..7a849db4d6 100644 --- a/api/net/netfilter.hpp +++ b/api/net/netfilter.hpp @@ -64,12 +64,11 @@ struct Filter_verdict { return std::move(packet); } }; -template -struct Inet; +class Inet; template using Packetfilter = - delegate(typename IPV::IP_packet_ptr, Inet&, Conntrack::Entry_ptr)>; + delegate(typename IPV::IP_packet_ptr, Inet&, Conntrack::Entry_ptr)>; /** * @brief A filter chain consisting of a list of packet filters. @@ -87,7 +86,7 @@ struct Filter_chain /** * Execute the chain */ - Filter_verdict operator()(IP_packet_ptr pckt, Inet& stack, Conntrack::Entry_ptr ct) + Filter_verdict operator()(IP_packet_ptr pckt, Inet& stack, Conntrack::Entry_ptr ct) { Filter_verdict verdict{std::move(pckt), Filter_verdict_type::ACCEPT}; int i = 0; diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index 33c93c1943..6979a1d168 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include namespace openssl { diff --git a/api/net/router b/api/net/router index 3ad2e33632..bfaf7f277f 100644 --- a/api/net/router +++ b/api/net/router @@ -24,7 +24,7 @@ namespace net { ? ip4::Addr{obj["nexthop"].GetString()} : 0; int N = obj["iface"].GetInt(); - auto& iface = Super_stack::get(N); + auto& iface = Super_stack::get(N); int cost = (not obj.HasMember("cost")) ? 100 : obj["cost"].GetInt(); diff --git a/api/net/router.hpp b/api/net/router.hpp index bacbb87be0..41ba8e3aea 100644 --- a/api/net/router.hpp +++ b/api/net/router.hpp @@ -34,7 +34,7 @@ namespace net { template struct Route { - using Stack = Inet; + using Stack = Inet; using Stack_ptr = Stack*; using Addr = typename IPV::addr; using Netmask = typename IPV::addr; @@ -112,7 +112,7 @@ namespace net { using Stack = typename Route::Stack; using Stack_ptr = typename Route::Stack_ptr; - using Forward_delg = typename Inet::Forward_delg; + using Forward_delg = typename Inet::Forward_delg; using Addr = typename IPV::addr; using Packet_ptr = typename IPV::IP_packet_ptr; using Interfaces = std::vector>; diff --git a/api/net/super_stack.hpp b/api/net/super_stack.hpp index b7d4af7841..b945fd81a2 100644 --- a/api/net/super_stack.hpp +++ b/api/net/super_stack.hpp @@ -20,7 +20,8 @@ #define NET_SUPER_STACK_HPP #include -#include "inet.hpp" +#include +#include #include #include #include @@ -39,15 +40,8 @@ struct Stack_not_found : public Super_stack_err class Super_stack { public: - template - using Inet_ptr = std::unique_ptr>; - template - using Stacks = std::map>; - template - using Stack_collection = std::vector>; - - using IP4_stack = Inet; - using IP4_stacks = Stack_collection; + using Stacks = std::map>; + using IP4_stacks = std::vector; public: static Super_stack& inet() @@ -56,11 +50,8 @@ class Super_stack { return stack_; } - template - static Inet& get(int N); - - template - static Inet& get(int N, int sub); + static Inet& get(int N); + static Inet& get(int N, int sub); /** * @brief Get a stack by MAC addr. @@ -68,15 +59,13 @@ class Super_stack { * * @param[in] mac The mac * - * @tparam IPV IP version + * @tparam IP version * * @return A stack */ - template - static Inet& get(const std::string& mac); + static Inet& get(const std::string& mac); - template - Inet& create(hw::Nic& nic, int N, int sub); + Inet& create(hw::Nic& nic, int N, int sub); /** * @brief Create a stack on the given Nic, @@ -84,12 +73,11 @@ class Super_stack { * * @param nic The nic * - * @tparam IPV IP version + * @tparam IP version * * @return A stack */ - template - Inet& create(hw::Nic& nic); + Inet& create(hw::Nic& nic); IP4_stacks& ip4_stacks() { return ip4_stacks_; } diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index facb1d1f3c..328e499046 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -27,12 +27,13 @@ #include // connections, listeners #include // writeq -#include #include #include namespace net { + class Inet; + struct TCP_error : public std::runtime_error { using runtime_error::runtime_error; }; @@ -384,8 +385,7 @@ namespace net { * * @return An IP4 address */ - tcp::Address address() const noexcept - { return inet_.ip_addr(); } + tcp::Address address() const noexcept; /** * @brief The stack object for which the TCP instance is "bound to" @@ -461,7 +461,7 @@ namespace net { Listeners listeners_; Connections connections_; - IPStack::Port_utils& ports_; + IP4::Port_utils& ports_; downstream _network_layer_out; @@ -539,8 +539,7 @@ namespace net { * * @return An IP4 object */ - IP4& network() const - { return inet_.ip_obj(); } + IP4& network() const; /** * @brief Drops the TCP segment @@ -597,8 +596,7 @@ namespace net { * * @return True if valid source, False otherwise. */ - bool is_valid_source(const tcp::Address addr) const noexcept - { return addr == 0 or inet_.is_valid_source(addr); } + bool is_valid_source(const tcp::Address addr) const noexcept; /** * @brief Try to find the listener bound to socket. @@ -705,8 +703,7 @@ namespace net { /** * @brief Force the TCP to process the it's queue with the current amount of available packets. */ - void kick() - { process_writeq(inet_.transmit_queue_available()); } + void kick(); }; // < class TCP diff --git a/api/net/vlan b/api/net/vlan index 1829698516..8b12f76a71 100644 --- a/api/net/vlan +++ b/api/net/vlan @@ -33,7 +33,7 @@ void parse_vlan_entry(const T& net) // Get the index (nic) the VLAN should be place upon auto iface = net["iface"].GetInt(); - auto& nic = Super_stack::get(iface).nic(); + auto& nic = Super_stack::get(iface).nic(); Expects(net.HasMember("vlan") && "VLAN config not found (\"vlan\")"); @@ -59,7 +59,7 @@ void parse_vlan_entry(const T& net) auto& vif = manager.add(nic, id); // Create stack on VLAN Nic [iface, id] - auto& stack = Super_stack::inet().create(vif, iface, id); + auto& stack = Super_stack::inet().create(vif, iface, id); // Configure the network stack Expects(entry.HasMember("address")); diff --git a/api/plugins/unik.hpp b/api/plugins/unik.hpp index f35fb29a40..66196d7a5e 100644 --- a/api/plugins/unik.hpp +++ b/api/plugins/unik.hpp @@ -18,7 +18,7 @@ #ifndef PLUGINS_UNIK_HPP #define PLUGINS_UNIK_HPP -#include +#include namespace unik{ @@ -28,7 +28,7 @@ namespace unik{ public: using Registered_event = delegate; - static void register_instance(net::Inet& inet, const net::UDP::port_t port = default_port); + static void register_instance(net::Inet& inet, const net::UDP::port_t port = default_port); static void register_instance_dhcp(); static void on_registered(Registered_event e) { on_registered_ = e; diff --git a/api/util/async.hpp b/api/util/async.hpp index 069a5d1487..4bd0feccfd 100644 --- a/api/util/async.hpp +++ b/api/util/async.hpp @@ -19,7 +19,7 @@ #ifndef UTIL_ASYNC_HPP #define UTIL_ASYNC_HPP -#include +#include #include #include diff --git a/examples/IRCd/ircd/ircd.hpp b/examples/IRCd/ircd/ircd.hpp index d4badcb0a5..980c216712 100644 --- a/examples/IRCd/ircd/ircd.hpp +++ b/examples/IRCd/ircd/ircd.hpp @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include diff --git a/examples/IRCd/service.cpp b/examples/IRCd/service.cpp index ebd1bb3a11..3b1f4bfe0e 100644 --- a/examples/IRCd/service.cpp +++ b/examples/IRCd/service.cpp @@ -106,7 +106,7 @@ void print_stats(int) printf("[%s] Conns/sec %.1f Heap %.1f kb\n", now().c_str(), cps, OS::heap_usage() / 1024.0); // client and channel stats - auto& inet = net::Inet4::stack<0>(); + auto& inet = net::Inet::stack<0>(); printf("Syns: %u Conns: %lu Users: %u RAM: %lu bytes Chans: %u\n", ircd->get_counter(STAT_TOTAL_CONNS), diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/service.cpp index 7a90edc681..9f0585d25a 100644 --- a/examples/LiveUpdate/service.cpp +++ b/examples/LiveUpdate/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include "liu.hpp" diff --git a/examples/TCP_perf/service.cpp b/examples/TCP_perf/service.cpp index 646db96075..17b02c1895 100644 --- a/examples/TCP_perf/service.cpp +++ b/examples/TCP_perf/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include #include diff --git a/examples/TLS_server/service.cpp b/examples/TLS_server/service.cpp index 9249a29ca3..68651e7026 100644 --- a/examples/TLS_server/service.cpp +++ b/examples/TLS_server/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include #include diff --git a/examples/UDP_perf/service.cpp b/examples/UDP_perf/service.cpp index 0a96985be4..6a22edb2ac 100644 --- a/examples/UDP_perf/service.cpp +++ b/examples/UDP_perf/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include #include diff --git a/examples/acorn/service.cpp b/examples/acorn/service.cpp index a2c9f0dcb6..72bd49e300 100644 --- a/examples/acorn/service.cpp +++ b/examples/acorn/service.cpp @@ -34,7 +34,7 @@ static std::unique_ptr logger_; static fs::Disk_ptr disk; #include -#include +#include static void start_acorn(net::Inet& inet) { diff --git a/examples/demo_linux/service.cpp b/examples/demo_linux/service.cpp index 4343ae9d09..d598bfedeb 100644 --- a/examples/demo_linux/service.cpp +++ b/examples/demo_linux/service.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include diff --git a/examples/demo_service/service.cpp b/examples/demo_service/service.cpp index 08ccd9ceda..5fa19761c8 100644 --- a/examples/demo_service/service.cpp +++ b/examples/demo_service/service.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include #include #include diff --git a/examples/mender/service.cpp b/examples/mender/service.cpp index b7be7faeaf..4a7fd08a0a 100644 --- a/examples/mender/service.cpp +++ b/examples/mender/service.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include @@ -44,7 +44,7 @@ void Service::start(const std::string&) // when generating private key (compute heavy) void Service::ready() { - auto& inet = net::Inet4::stack(); + auto& inet = net::Inet::stack(); inet.network_config( { 10, 0, 0, 42 }, // IP { 255,255,255, 0 }, // Netmask diff --git a/examples/scoped_profiler/service.cpp b/examples/scoped_profiler/service.cpp index a9d9ed8d25..a6da41124e 100644 --- a/examples/scoped_profiler/service.cpp +++ b/examples/scoped_profiler/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include #include diff --git a/examples/syslog/service.cpp b/examples/syslog/service.cpp index bf846e2587..4ddedf88fc 100644 --- a/examples/syslog/service.cpp +++ b/examples/syslog/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include /* For Posix */ #include @@ -29,9 +29,9 @@ extern "C" int main(); void Service::start(const std::string&) { // DHCP on interface 0 - auto& inet = net::Inet4::ifconfig(10.0); + auto& inet = net::Inet::ifconfig(10.0); // static IP in case DHCP fails - net::Inet4::ifconfig({ 10, 0, 0, 45 }, // IP + net::Inet::ifconfig({ 10, 0, 0, 45 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 1 }, // Gateway { 10, 0, 0, 1} ); // DNS diff --git a/examples/tcp/service.cpp b/examples/tcp/service.cpp index b018f514a1..ebba165191 100644 --- a/examples/tcp/service.cpp +++ b/examples/tcp/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include /** * An example to show incoming and outgoing TCP Connections. diff --git a/examples/websocket/service.cpp b/examples/websocket/service.cpp index 307abac117..4d3a281abc 100644 --- a/examples/websocket/service.cpp +++ b/examples/websocket/service.cpp @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include @@ -55,7 +55,7 @@ std::unique_ptr server; void Service::start() { // Retreive the stack (configured from outside) - auto& inet = net::Inet4::stack<0>(); + auto& inet = net::Inet::stack<0>(); Expects(inet.is_configured()); // Init the memdisk diff --git a/lib/LiveUpdate/serialize_tcp.cpp b/lib/LiveUpdate/serialize_tcp.cpp index d327d6844d..c6a1f11e1e 100644 --- a/lib/LiveUpdate/serialize_tcp.cpp +++ b/lib/LiveUpdate/serialize_tcp.cpp @@ -18,14 +18,14 @@ * by Alf-Andre Walla 2016-2017 * **/ -#include +#include #include #include "serialize_tcp.hpp" #include #include using namespace net::tcp; -static std::unordered_set*> slumbering_ip4; +static std::unordered_set slumbering_ip4; Connection::State* serialized_tcp::to_state(int8_t state) const { diff --git a/lib/mana/examples/simple/service.cpp b/lib/mana/examples/simple/service.cpp index 0d861d2dc6..d544771717 100644 --- a/lib/mana/examples/simple/service.cpp +++ b/lib/mana/examples/simple/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include using namespace mana; @@ -27,7 +27,7 @@ std::unique_ptr server; void Service::start(const std::string&) { // Setup stack; try DHCP - auto& stack = net::Inet4::ifconfig(3.0); + auto& stack = net::Inet::ifconfig(3.0); // Static config stack.network_config({ 10,0,0,42 }, // IP { 255,255,255,0 }, // Netmask diff --git a/lib/microLB/micro_lb/autoconf.cpp b/lib/microLB/micro_lb/autoconf.cpp index 751af9a87b..25b84bf41d 100644 --- a/lib/microLB/micro_lb/autoconf.cpp +++ b/lib/microLB/micro_lb/autoconf.cpp @@ -17,7 +17,7 @@ namespace microLB auto& clients = obj["clients"]; // client network interface const int CLIENT_NET = clients["iface"].GetInt(); - auto& netinc = net::Super_stack::get(CLIENT_NET); + auto& netinc = net::Super_stack::get(CLIENT_NET); // client port const int CLIENT_PORT = clients["port"].GetUint(); assert(CLIENT_PORT > 0 && CLIENT_PORT < 65536); @@ -28,7 +28,7 @@ namespace microLB auto& nodes = obj["nodes"]; const int NODE_NET = nodes["iface"].GetInt(); - auto& netout = net::Super_stack::get(NODE_NET); + auto& netout = net::Super_stack::get(NODE_NET); netout.tcp().set_MSL(15s); // create load balancer diff --git a/lib/microLB/micro_lb/balancer.hpp b/lib/microLB/micro_lb/balancer.hpp index 7eda3889cf..cd8d4dedda 100644 --- a/lib/microLB/micro_lb/balancer.hpp +++ b/lib/microLB/micro_lb/balancer.hpp @@ -1,10 +1,10 @@ #pragma once -#include +#include #include namespace microLB { - typedef net::Inet netstack_t; + typedef net::Inet netstack_t; typedef net::tcp::Connection_ptr tcp_ptr; typedef std::vector queue_vector_t; typedef delegate pool_signal_t; diff --git a/lib/uplink/config.cpp b/lib/uplink/config.cpp index 33f609cc56..9527183871 100644 --- a/lib/uplink/config.cpp +++ b/lib/uplink/config.cpp @@ -18,6 +18,7 @@ #include #include "config.hpp" #include "common.hpp" +#include #include #ifndef RAPIDJSON_HAS_STDSTRING @@ -65,17 +66,17 @@ namespace uplink { if(index.IsNumber()) { - config_.inet = &net::Super_stack::get(index.GetInt()); + config_.inet = &net::Super_stack::get(index.GetInt()); } else { - config_.inet = &net::Super_stack::get(index.GetString()); + config_.inet = &net::Super_stack::get(index.GetString()); } } // If not given, pick the first stack else { - config_.inet = &net::Super_stack::get(0); + config_.inet = &net::Super_stack::get(0); } // Reboot on panic (optional) diff --git a/lib/uplink/config.hpp b/lib/uplink/config.hpp index 3879154861..cee710d3df 100644 --- a/lib/uplink/config.hpp +++ b/lib/uplink/config.hpp @@ -20,14 +20,13 @@ #define UPLINK_CONFIG_HPP #include -#include #include namespace uplink { struct Config { - net::Inet* inet; + net::Inet *inet; std::string url; std::string token; std::string tag; diff --git a/lib/uplink/starbase/service.cpp b/lib/uplink/starbase/service.cpp index 41d55d181a..ffb475639f 100644 --- a/lib/uplink/starbase/service.cpp +++ b/lib/uplink/starbase/service.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include #include #include diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index a7b52bb998..7d9df216ed 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -83,7 +83,7 @@ namespace uplink { } } - void WS_uplink::start(net::Inet& inet) { + void WS_uplink::start(net::Inet& inet) { MYINFO("Starting WS uplink on %s with ID %s", inet.ifname().c_str(), id_.c_str()); diff --git a/lib/uplink/ws_uplink.hpp b/lib/uplink/ws_uplink.hpp index b6369d6bac..57565e0c42 100644 --- a/lib/uplink/ws_uplink.hpp +++ b/lib/uplink/ws_uplink.hpp @@ -22,7 +22,7 @@ #include "transport.hpp" #include "config.hpp" -#include +#include #include #include #include @@ -39,7 +39,7 @@ class WS_uplink { WS_uplink(Config config); - void start(net::Inet&); + void start(net::Inet&); void auth(); @@ -69,7 +69,7 @@ class WS_uplink { private: Config config_; - net::Inet& inet_; + net::Inet& inet_; std::unique_ptr client_; net::WebSocket_ptr ws_; std::string id_; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2b0270a236..f9db1a1ed2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,8 +48,9 @@ set(OS_OBJECTS net/tcp/tcp.cpp net/tcp/connection.cpp net/tcp/connection_states.cpp net/tcp/write_queue.cpp net/tcp/rttm.cpp net/tcp/listener.cpp net/tcp/read_buffer.cpp net/ip4/icmp4.cpp net/ip4/udp.cpp net/ip4/udp_socket.cpp + net/ip6/ip6.cpp net/dns/dns.cpp net/dns/client.cpp net/dhcp/dh4client.cpp net/dhcp/dhcpd.cpp - net/buffer_store.cpp net/inet4.cpp + net/buffer_store.cpp net/inet.cpp net/super_stack.cpp net/configure.cpp net/conntrack.cpp net/vlan_manager.cpp net/http/header.cpp net/http/header_fields.cpp net/http/message.cpp net/http/request.cpp net/http/response.cpp net/http/status_codes.cpp net/http/time.cpp net/http/version.cpp diff --git a/src/include/sockfd.hpp b/src/include/sockfd.hpp index 654bdda05e..099293a90f 100644 --- a/src/include/sockfd.hpp +++ b/src/include/sockfd.hpp @@ -20,7 +20,7 @@ #define INCLUDE_SOCKFD_HPP #include -#include +#include #include "fd.hpp" class SockFD : public FD { diff --git a/src/include/udp_fd.hpp b/src/include/udp_fd.hpp index 17c0e3aa87..5f7cea459e 100644 --- a/src/include/udp_fd.hpp +++ b/src/include/udp_fd.hpp @@ -20,7 +20,7 @@ #define INCLUDE_UDP_FD_HPP #include "sockfd.hpp" -#include +#include #include #include diff --git a/src/kernel/terminal.cpp b/src/kernel/terminal.cpp index 50505b26cd..38d9018012 100644 --- a/src/kernel/terminal.cpp +++ b/src/kernel/terminal.cpp @@ -15,6 +15,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include #include #include #include diff --git a/src/net/configure.cpp b/src/net/configure.cpp index 24b618797c..f965141b0f 100644 --- a/src/net/configure.cpp +++ b/src/net/configure.cpp @@ -17,6 +17,7 @@ #include +#include #include #include @@ -43,7 +44,7 @@ Addresses parse_iface(T& obj) return addresses; } -inline void config_stack(Inet& stack, const Addresses& addrs) +inline void config_stack(Inet& stack, const Addresses& addrs) { if(addrs.empty()) { MYINFO("! WARNING: No config for stack %s", stack.ifname().c_str()); @@ -76,7 +77,7 @@ void configure(const rapidjson::Value& net) auto N = val["iface"].GetInt(); - auto& stack = Super_stack::get(N); + auto& stack = Super_stack::get(N); // if config is not set, just ignore if(not val.HasMember("config")) { diff --git a/src/net/dhcp/dh4client.cpp b/src/net/dhcp/dh4client.cpp index 46568968fc..d987bd2f4d 100644 --- a/src/net/dhcp/dh4client.cpp +++ b/src/net/dhcp/dh4client.cpp @@ -26,6 +26,7 @@ #define MYINFO(X,...) INFO("DHCPv4",X,##__VA_ARGS__) #include +#include #include #include #include diff --git a/src/net/dhcp/dhcpd.cpp b/src/net/dhcp/dhcpd.cpp index e1a88e3f44..3958f6611b 100644 --- a/src/net/dhcp/dhcpd.cpp +++ b/src/net/dhcp/dhcpd.cpp @@ -15,6 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. #include +#include #include using namespace net; diff --git a/src/net/dns/client.cpp b/src/net/dns/client.cpp index 119598a7ec..597e8a68cd 100644 --- a/src/net/dns/client.cpp +++ b/src/net/dns/client.cpp @@ -16,6 +16,7 @@ // limitations under the License. #include +#include namespace net { diff --git a/src/net/http/server.cpp b/src/net/http/server.cpp index de4cee210f..14ae9ba616 100644 --- a/src/net/http/server.cpp +++ b/src/net/http/server.cpp @@ -16,6 +16,7 @@ // limitations under the License. #include +#include #include namespace http { diff --git a/src/net/inet4.cpp b/src/net/inet.cpp similarity index 73% rename from src/net/inet4.cpp rename to src/net/inet.cpp index 9802aa4f90..fa4683fcc0 100644 --- a/src/net/inet4.cpp +++ b/src/net/inet.cpp @@ -15,19 +15,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include #include using namespace net; -Inet4::Inet4(hw::Nic& nic) +Inet::Inet(hw::Nic& nic) : ip4_addr_(IP4::ADDR_ANY), netmask_(IP4::ADDR_ANY), gateway_(IP4::ADDR_ANY), dns_server_(IP4::ADDR_ANY), - nic_(nic), arp_(*this), ip4_(*this), + nic_(nic), arp_(*this), ip4_(*this), ip6_(*this), icmp_(*this), udp_(*this), tcp_(*this), dns_(*this), domain_name_{}, MTU_(nic.MTU()) @@ -36,19 +36,20 @@ Inet4::Inet4(hw::Nic& nic) /** SMP related **/ this->cpu_id = SMP::cpu_id(); - INFO("Inet4", "Bringing up %s on CPU %d", + INFO("Inet", "Bringing up %s on CPU %d", ifname().c_str(), this->get_cpu_id()); /** Upstream delegates */ auto arp_bottom(upstream{arp_, &Arp::receive}); auto ip4_bottom(upstream_ip{ip4_, &IP4::receive}); + auto ip6_bottom(upstream_ip{ip6_, &IP6::receive}); auto icmp4_bottom(upstream{icmp_, &ICMPv4::receive}); auto udp4_bottom(upstream{udp_, &UDP::receive}); auto tcp_bottom(upstream{tcp_, &TCP::receive}); /** Upstream wiring */ // Packets available - nic.on_transmit_queue_available({this, &Inet4::process_sendq}); + nic.on_transmit_queue_available({this, &Inet::process_sendq}); // Link -> Arp nic_.set_arp_upstream(arp_bottom); @@ -56,6 +57,9 @@ Inet4::Inet4(hw::Nic& nic) // Link -> IP4 nic_.set_ip4_upstream(ip4_bottom); + // Link -> IP6 + nic_.set_ip6_upstream(ip6_bottom); + // IP4 -> ICMP ip4_.set_icmp_handler(icmp4_bottom); @@ -69,6 +73,7 @@ Inet4::Inet4(hw::Nic& nic) auto link_top(nic_.create_link_downstream()); auto arp_top(IP4::downstream_arp{arp_, &Arp::transmit}); auto ip4_top(downstream{ip4_, &IP4::transmit}); + auto ip6_top(downstream{ip6_, &IP6::transmit}); /** Downstream wiring. */ @@ -84,9 +89,18 @@ Inet4::Inet4(hw::Nic& nic) // IP4 -> Arp ip4_.set_linklayer_out(arp_top); + // ICMP6 -> IP6 + // icmp6->set_network_out(ip6_top); + // UDP6 -> IP6 + // udp6->set_network_out(ip6_top); + // TCP6 -> IP6 + // tcp6->set_network_out(ip6_top); + // Arp -> Link + // IP6 -> Link assert(link_top); arp_.set_linklayer_out(link_top); + //ip6_.set_linklayer_out(link_top); #ifndef INCLUDEOS_SINGLE_THREADED // move this nework stack automatically @@ -97,7 +111,7 @@ Inet4::Inet4(hw::Nic& nic) #endif } -void Inet4::error_report(Error& err, Packet_ptr orig_pckt) { +void Inet::error_report(Error& err, Packet_ptr orig_pckt) { auto pckt_ip4 = static_unique_ptr_cast(std::move(orig_pckt)); bool too_big = false; @@ -161,8 +175,8 @@ void Inet4::error_report(Error& err, Packet_ptr orig_pckt) { } } -void Inet4::negotiate_dhcp(double timeout, dhcp_timeout_func handler) { - INFO("Inet4", "Negotiating DHCP (%.1fs timeout)...", timeout); +void Inet::negotiate_dhcp(double timeout, dhcp_timeout_func handler) { + INFO("Inet", "Negotiating DHCP (%.1fs timeout)...", timeout); if (!dhcp_) dhcp_ = std::make_shared(*this); // @timeout for DHCP-server negotation @@ -172,34 +186,47 @@ void Inet4::negotiate_dhcp(double timeout, dhcp_timeout_func handler) { dhcp_->on_config(handler); } -void Inet4::network_config(IP4::addr addr, +void Inet::network_config(IP4::addr addr, IP4::addr nmask, IP4::addr gateway, - IP4::addr dns) + IP4::addr dns, + IP6::addr addr6, + IP6::addr prefix6, + IP6::addr gateway6) { this->ip4_addr_ = addr; this->netmask_ = nmask; this->gateway_ = gateway; this->dns_server_ = (dns == IP4::ADDR_ANY) ? gateway : dns; - INFO("Inet4", "Network configured (%s)", nic_.mac().to_string().c_str()); + INFO("Inet", "Network configured (%s)", nic_.mac().to_string().c_str()); INFO2("IP: \t\t%s", ip4_addr_.str().c_str()); INFO2("Netmask: \t%s", netmask_.str().c_str()); INFO2("Gateway: \t%s", gateway_.str().c_str()); INFO2("DNS Server: \t%s", dns_server_.str().c_str()); + if (addr6 != IP6::ADDR_ANY) { + this->ip6_addr_ = addr6; + this->ip6_prefix_ = prefix6; + this->ip6_gateway_ = gateway6; + INFO("Inet6", "Network configured (%s)", nic_.mac().to_string().c_str()); + INFO2("IP6: \t\t%s", ip6_addr_.str().c_str()); + INFO2("Prefix: \t%s", ip6_prefix_.str().c_str()); + INFO2("Gateway: \t%s", ip6_gateway_.str().c_str()); + } + for(auto& handler : configured_handlers_) handler(*this); configured_handlers_.clear(); } -void Inet4::enable_conntrack(std::shared_ptr ct) +void Inet::enable_conntrack(std::shared_ptr ct) { Expects(conntrack_ == nullptr && "Conntrack is already set"); conntrack_ = ct; } -void Inet4::process_sendq(size_t packets) { +void Inet::process_sendq(size_t packets) { //////////////////////////////////////////// // divide up fairly @@ -242,14 +269,44 @@ void Inet4::process_sendq(size_t packets) { */ } -void Inet4::force_start_send_queues() +void Inet::force_start_send_queues() { size_t packets = transmit_queue_available(); if (packets) process_sendq(packets); } -void Inet4::move_to_this_cpu() +void Inet::move_to_this_cpu() { this->cpu_id = SMP::cpu_id(); nic_.move_to_this_cpu(); } + +void Inet::cache_link_addr(IP4::addr ip, MAC::Addr mac) +{ arp_.cache(ip, mac); } + +void Inet::flush_link_cache() +{ arp_.flush_cache(); } + +void Inet::set_link_cache_flush_interval(std::chrono::minutes min) +{ arp_.set_cache_flush_interval(min); } + +void Inet::set_route_checker(Route_checker delg) +{ arp_.set_proxy_policy(delg); } + +void Inet::reset_pmtu(Socket dest, IP4::PMTU pmtu) +{ tcp_.reset_pmtu(dest, pmtu); /* Maybe later: udp_.reset_pmtu(dest, pmtu);*/ } + +void Inet::resolve(const std::string& hostname, + resolve_func func, + bool force) +{ + dns_.resolve(this->dns_server_, hostname, func, force); +} + +void Inet::resolve(const std::string& hostname, + IP4::addr server, + resolve_func func, + bool force) +{ + dns_.resolve(server, hostname, func, force); +} diff --git a/src/net/ip4/arp.cpp b/src/net/ip4/arp.cpp index a83eafa6b3..64f16e8fb7 100644 --- a/src/net/ip4/arp.cpp +++ b/src/net/ip4/arp.cpp @@ -24,7 +24,7 @@ #include -#include +#include #include #include #include diff --git a/src/net/ip4/icmp4.cpp b/src/net/ip4/icmp4.cpp index f8fc707a1a..1d710d773e 100644 --- a/src/net/ip4/icmp4.cpp +++ b/src/net/ip4/icmp4.cpp @@ -17,6 +17,7 @@ // #define DEBUG #include +#include namespace net { @@ -176,7 +177,7 @@ namespace net { } void ICMPv4::ping(const std::string& hostname, icmp_func callback, int sec_wait) { - inet_.resolve(hostname, Inet::resolve_func::make_packed([this, callback, sec_wait] (IP4::addr a, Error err) { + inet_.resolve(hostname, Inet::resolve_func::make_packed([this, callback, sec_wait] (IP4::addr a, Error err) { if (!err and a != IP4::ADDR_ANY) ping(a, callback, sec_wait); })); diff --git a/src/net/ip4/ip4.cpp b/src/net/ip4/ip4.cpp index 439761a868..5951de8dc1 100644 --- a/src/net/ip4/ip4.cpp +++ b/src/net/ip4/ip4.cpp @@ -23,6 +23,7 @@ #endif #include +#include #include #include #include @@ -437,4 +438,14 @@ namespace net { } } + uint16_t IP4::MDDS() const + { return stack_.MTU() - sizeof(ip4::Header); } + + uint16_t IP4::default_PMTU() const noexcept + { return stack_.MTU(); } + + const ip4::Addr IP4::local_ip() const { + return stack_.ip_addr(); + } + } //< namespace net diff --git a/src/net/ip4/udp.cpp b/src/net/ip4/udp.cpp index c58350ff80..b023b0b105 100644 --- a/src/net/ip4/udp.cpp +++ b/src/net/ip4/udp.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -311,4 +312,19 @@ namespace net { } } + ip4::Addr UDP::local_ip() const + { return stack_.ip_addr(); } + + UDPSocket& UDP::bind(port_t port) + { return bind({stack_.ip_addr(), port}); } + + UDPSocket& UDP::bind() + { return bind(stack_.ip_addr()); } + + bool UDP::is_bound(const port_t port) const + { return is_bound({stack_.ip_addr(), port}); } + + uint16_t UDP::max_datagram_size() noexcept + { return stack().ip_obj().MDDS() - sizeof(header); } + } //< namespace net diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index f6cfb0aace..7db82a824c 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -15,163 +15,311 @@ // See the License for the specific language governing permissions and // limitations under the License. -//#define DEBUG +//#define IP6_DEBUG 1 +#ifdef IP6_DEBUG +#define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define PRINT(fmt, ...) /* fmt */ +#endif + #include +#include #include - -#include +#include +#include +#include namespace net { - const IP6::addr IP6::addr::node_all_nodes(0xFF01, 0, 0, 0, 0, 0, 0, 1); - const IP6::addr IP6::addr::node_all_routers(0xFF01, 0, 0, 0, 0, 0, 0, 2); - const IP6::addr IP6::addr::node_mDNSv6(0xFF01, 0, 0, 0, 0, 0, 0, 0xFB); + const ip6::Addr ip6::Addr::node_all_nodes(0xFF01, 0, 0, 0, 0, 0, 0, 1); + const ip6::Addr ip6::Addr::node_all_routers(0xFF01, 0, 0, 0, 0, 0, 0, 2); + const ip6::Addr ip6::Addr::node_mDNSv6(0xFF01, 0, 0, 0, 0, 0, 0, 0xFB); - const IP6::addr IP6::addr::link_unspecified(0, 0, 0, 0, 0, 0, 0, 0); + const ip6::Addr ip6::Addr::link_unspecified(0, 0, 0, 0, 0, 0, 0, 0); - const IP6::addr IP6::addr::link_all_nodes(0xFF02, 0, 0, 0, 0, 0, 0, 1); - const IP6::addr IP6::addr::link_all_routers(0xFF02, 0, 0, 0, 0, 0, 0, 2); - const IP6::addr IP6::addr::link_mDNSv6(0xFF02, 0, 0, 0, 0, 0, 0, 0xFB); + const ip6::Addr ip6::Addr::link_all_nodes(0xFF02, 0, 0, 0, 0, 0, 0, 1); + const ip6::Addr ip6::Addr::link_all_routers(0xFF02, 0, 0, 0, 0, 0, 0, 2); + const ip6::Addr ip6::Addr::link_mDNSv6(0xFF02, 0, 0, 0, 0, 0, 0, 0xFB); - const IP6::addr IP6::addr::link_dhcp_servers(0xFF02, 0, 0, 0, 0, 0, 0x01, 0x02); - const IP6::addr IP6::addr::site_dhcp_servers(0xFF05, 0, 0, 0, 0, 0, 0x01, 0x03); - - IP6::IP6(const IP6::addr& lo) - : local(lo) - { - assert(sizeof(addr) == 16); - assert(sizeof(header) == 40); - assert(sizeof(options_header) == 8); - } + const ip6::Addr ip6::Addr::link_dhcp_servers(0xFF02, 0, 0, 0, 0, 0, 0x01, 0x02); + const ip6::Addr ip6::Addr::site_dhcp_servers(0xFF05, 0, 0, 0, 0, 0, 0x01, 0x03); - uint8_t IP6::parse6(uint8_t*& reader, uint8_t next) - { - switch (next) - { - case PROTO_HOPOPT: - case PROTO_OPTSv6: - { - debug(">>> IPv6 options header %s", protocol_name(next).c_str()); - - options_header& opts = *(options_header*) reader; - reader += opts.size(); - - debug("OPTSv6 size: %d\n", opts.size()); - debug("OPTSv6 ext size: %d\n", opts.extended()); - - next = opts.next(); - debug("OPTSv6 next: %s\n", protocol_name(next).c_str()); - } break; - case PROTO_ICMPv6: - break; - case PROTO_UDP: - break; - - default: - debug("Not parsing: %s\n", protocol_name(next).c_str()); - } + const IP6::addr IP6::ADDR_ANY(0, 0, 0, 0); + const IP6::addr IP6::ADDR_LOOPBACK(0, 0, 0, 1); - return next; - } + IP6::IP6(Stack& inet) noexcept : + packets_rx_ {Statman::get().create(Stat::UINT64, inet.ifname() + ".ip6.packets_rx").get_uint64()}, + packets_tx_ {Statman::get().create(Stat::UINT64, inet.ifname() + ".ip6.packets_tx").get_uint64()}, + packets_dropped_ {Statman::get().create(Stat::UINT32, inet.ifname() + ".ip6.packets_dropped").get_uint32()}, + stack_ {inet} + {} - void IP6::bottom(Packet_ptr pckt) - { - debug(">>> IPv6 packet:"); + IP6::IP_packet_ptr IP6::drop(IP_packet_ptr ptr, Direction direction, Drop_reason reason) { + packets_dropped_++; + if (drop_handler_) + drop_handler_(std::move(ptr), direction, reason); + return nullptr; + } - uint8_t* reader = pckt->buffer(); - full_header& full = *(full_header*) reader; - reader += sizeof(full_header); + IP6::IP_packet_ptr IP6::drop_invalid_in(IP6::IP_packet_ptr packet) + { + IP6::Direction up = IP6::Direction::Upstream; - header& hdr = full.ip6_hdr; - /* - debug("IPv6 v: %i \t class: %i size: %i \n", - (int) hdr.version(), (int) hdr.tclass(), hdr.size() )"; - */ - uint8_t next = hdr.next(); + if (UNLIKELY(not packet->is_ipv6())) + return drop(std::move(packet), up, Drop_reason::Wrong_version); - while (next != PROTO_NoNext) - { - auto it = proto_handlers.find(next); - if (it != proto_handlers.end()) - { - // forward packet to handler - pckt->set_payload(reader); - it->second(pckt); - } - else - // just print information - next = parse6(reader, next); - } - - }; + /* TODO: Add more checks */ + return packet; + } - static const std::string lut = "0123456789abcdef"; + IP6::IP_packet_ptr IP6::drop_invalid_out(IP6::IP_packet_ptr packet) + { + if (packet->ip_dst().is_loopback()) { + drop(std::move(packet), Direction::Downstream, Drop_reason::Bad_destination); + } + return packet; + } - std::string IP6::addr::str() const + /* TODO: Check RFC */ + bool IP6::is_for_me(ip6::Addr dst) const { - std::string ret(40, 0); - int counter = 0; + return stack_.is_valid_source(dst) + or local_ip() == ADDR_ANY; + } - const uint8_t* octet = i8; + void IP6::ipv6_ext_header_receive(Packet_ptr pckt) + { + auto packet = static_unique_ptr_cast(std::move(pckt)); + uint8_t* reader = packet->buf(); + auto header = packet->ip6_header(); + reader += sizeof(header); + ip6::ExtensionHeader& ext = *(ip6::ExtensionHeader*)reader; + auto next_proto = packet->next_protocol(); + + while (next_proto != Protocol::IPv6_NONXT) { + if (next_proto == Protocol::HOPOPT) { + } else if (next_proto == Protocol::OPTSV6) { + } else { + PRINT("Unknown IPv6 extension header\n"); + break; + } + ext = *(ip6::ExtensionHeader*)reader; + reader += ext.size(); + next_proto = ext.next(); + } + } - for (int i = 0; i < 16; i++) + void IP6::receive(Packet_ptr pckt, const bool link_bcast) + { + // Cast to IP6 Packet + auto packet = static_unique_ptr_cast(std::move(pckt)); + + PRINT(" Source IP: %s Dest.IP: %s Type: %d LinkBcast: %d ", + packet->ip_src().str().c_str(), + packet->ip_dst().str().c_str(), + (int) packet->next_header(), + link_bcast); + + switch (packet->next_protocol()) { + case Protocol::HOPOPT: + PRINT("Type: ICMP6 hop by hop option\n"); break; + case Protocol::OPTSV6: + PRINT("Type: ICMP6 option: %d\n", (int)packet->next_protocol()); break; + case Protocol::IPv6_NONXT: + PRINT("Type: ICMP6 No Next\n"); break; + case Protocol::ICMPv6: + PRINT("Type: ICMP6\n"); break; + case Protocol::IPv4: + PRINT("Type: IPv4\n"); break; + case Protocol::UDP: + PRINT("Type: UDP\n"); break; + case Protocol::TCP: + PRINT("Type: TCP\n"); break; + default: + PRINT("Type: UNKNOWN %hhu. Dropping. \n", packet->next_protocol()); + } + + // Stat increment packets received + packets_rx_++; + + packet = drop_invalid_in(std::move(packet)); + if (UNLIKELY(packet == nullptr)) return; + + /* PREROUTING */ + // Track incoming packet if conntrack is active + Conntrack::Entry_ptr ct = nullptr; + +#if 0 + Conntrack::Entry_ptr ct = (stack_.conntrack()) + ? stack_.conntrack()->in(*packet) : nullptr; + auto res = prerouting_chain_(std::move(packet), stack_, ct); + if (UNLIKELY(res == Filter_verdict_type::DROP)) return; + + Ensures(res.packet != nullptr); + packet = res.release(); +#endif + + // Drop / forward if my ip address doesn't match dest. + if(not is_for_me(packet->ip_dst())) + { + // Forwarding disabled + if (not forward_packet_) { - ret[counter++] = lut[(octet[i] & 0xF0) >> 4]; - ret[counter++] = lut[(octet[i] & 0x0F) >> 0]; - if (i & 1) - ret[counter++] = ':'; + PRINT("Dropping packet \n"); + drop(std::move(packet), Direction::Upstream, Drop_reason::Bad_destination); } - ret.resize(counter-1); - return ret; + // Forwarding enabled + else + { + PRINT("Forwarding packet \n"); + forward_packet_(std::move(packet), stack_, ct); + } + return; + } + + PRINT("* Packet was for me\n"); + + /* INPUT */ + // Confirm incoming packet if conntrack is active +#if 0 + auto& conntrack = stack_.conntrack(); + if(conntrack) { + ct = (ct != nullptr) ? + conntrack->confirm(ct->second, ct->proto) : conntrack->confirm(*packet); + } + if(stack_.conntrack()) + stack_.conntrack()->confirm(*packet); // No need to set ct again + res = input_chain_(std::move(packet), stack_, ct); + if (UNLIKELY(res == Filter_verdict_type::DROP)) return; + + Ensures(res.packet != nullptr); + packet = res.release(); +#endif + + // Pass packet to it's respective protocol controller + switch (packet->next_protocol()) { + case Protocol::HOPOPT: + case Protocol::OPTSV6: + ipv6_ext_header_receive(std::move(pckt)); + break; + case Protocol::IPv6_NONXT: + /* Nothing after the icmp header */ + break; + case Protocol::ICMPv6: + icmp_handler_(std::move(packet)); + break; + case Protocol::UDP: + udp_handler_(std::move(packet)); + break; + case Protocol::TCP: + tcp_handler_(std::move(packet)); + break; + default: + // Send ICMP error of type Destination Unreachable and code PROTOCOL + // @note: If dest. is broadcast or multicast it should be dropped by now + //stack_.icmp().destination_unreachable(std::move(packet), icmp6::code::Dest_unreachable::PROTOCOL); + + drop(std::move(packet), Direction::Upstream, Drop_reason::Unknown_proto); + break; + } } - void IP6::transmit(std::shared_ptr& ip6_packet) - { - auto packet = *reinterpret_cast*> (&ip6_packet); + void IP6::transmit(Packet_ptr pckt) { + assert((size_t)pckt->size() > sizeof(header)); - //debug(" Transmitting %li b, from %s -> %s\n", - // pckt->len(), hdr.src.str().c_str(), hdr.dst.str().c_str()); + auto packet = static_unique_ptr_cast(std::move(pckt)); - _linklayer_out(packet); + /* + * RFC 1122 p. 30 + * When a host sends any datagram, the IP source address MUST + be one of its own IP addresses (but not a broadcast or + multicast address). + */ + if (UNLIKELY(not stack_.is_valid_source(packet->ip_src()))) { + drop(std::move(packet), Direction::Downstream, Drop_reason::Bad_source); + return; + } + + packet->make_flight_ready(); + + Conntrack::Entry_ptr ct = nullptr; +#if 0 + /* OUTPUT */ + Conntrack::Entry_ptr ct = + (stack_.conntrack()) ? stack_.conntrack()->in(*packet) : nullptr; + auto res = output_chain_(std::move(packet), stack_, ct); + if (UNLIKELY(res == Filter_verdict_type::DROP)) return; + + Ensures(res.packet != nullptr); + packet = res.release(); +#endif + + if (forward_packet_) { + forward_packet_(std::move(packet), stack_, ct); + return; + } + ship(std::move(packet), IP6::ADDR_ANY, ct); } - std::shared_ptr IP6::create(uint8_t proto, - Ethernet::addr ether_dest, const IP6::addr& ip6_dest) + void IP6::ship(Packet_ptr pckt, addr next_hop, Conntrack::Entry_ptr ct) { - // arbitrarily big buffer - uint8_t* data = new uint8_t[1580]; - - // @WARNING: Initializing packet as "full", i.e. size == capacity - Packet* packet = (Packet*) data; - new (packet) Packet(sizeof(data), 0, nullptr); - - IP6::full_header& full = *(IP6::full_header*) packet->buffer(); - // people dont think that it be, but it do - full.eth_hdr.type = Ethernet::ETH_IP6; - full.eth_hdr.dest = ether_dest; - - IP6::header& hdr = full.ip6_hdr; - - // set IPv6 packet parameters - hdr.src = addr::link_unspecified; - hdr.dst = ip6_dest; - // default header frame - hdr.init_scan0(); - // protocol for next header - hdr.set_next(proto); - // default hoplimit - hdr.set_hoplimit(64); - - // common offset of payload - packet->set_payload(packet->buffer() + sizeof(IP6::full_header)); - - auto ip6_packet = - std::shared_ptr ((PacketIP6*) packet, - [] (void* ptr) { delete[] (uint8_t*) ptr; }); - // now, free to use :) - return ip6_packet; + auto packet = static_unique_ptr_cast(std::move(pckt)); + + // Send loopback packets right back + if (UNLIKELY(stack_.is_valid_source(packet->ip_dst()))) { + PRINT(" Destination address is loopback \n"); + IP6::receive(std::move(packet), false); + return; + } + + // Filter illegal egress packets + packet = drop_invalid_out(std::move(packet)); + if (packet == nullptr) return; + + if (next_hop == IP6::ADDR_ANY) { +#if 0 + if (UNLIKELY(packet->ip_dst() == IP6::ADDR_BCAST)) { + next_hop = IP6::ADDR_BCAST; + } + else { +#endif + // Create local and target subnets + addr target = packet->ip_dst() & stack_.netmask6(); + addr local = stack_.ip6_addr() & stack_.netmask6(); + + // Compare subnets to know where to send packet + next_hop = target == local ? packet->ip_dst() : stack_.gateway6(); + + PRINT(" Next hop for %s, (netmask %s, local IP: %s, gateway: %s) == %s\n", + packet->ip_dst().str().c_str(), + stack_.netmask6().str().c_str(), + stack_.ip6_addr().str().c_str(), + stack_.gateway6().str().c_str(), + next_hop.str().c_str()); + + if(UNLIKELY(next_hop == IP6::ADDR_ANY)) { + PRINT(" Next_hop calculated to 0 (gateway == %s), dropping\n", + stack_.gateway6().str().c_str()); + drop(std::move(packet), Direction::Downstream, Drop_reason::Bad_destination); + return; + } + } + + // Stat increment packets transmitted + packets_tx_++; + +#if 0 + extern MAC::Addr linux_tap_device; + MAC::Addr dest_mac = linux_tap_device; + + PRINT(" Transmitting packet, layer begin: buf + %li\n", packet->layer_begin() - packet->buf()); + linklayer_out_(std::move(packet), dest_mac, Ethertype::IP6); +#endif } + const ip6::Addr IP6::local_ip() const { + return stack_.ip6_addr(); + } } diff --git a/src/net/nat/napt.cpp b/src/net/nat/napt.cpp index ffea29adf6..5fada464b4 100644 --- a/src/net/nat/napt.cpp +++ b/src/net/nat/napt.cpp @@ -17,6 +17,7 @@ #include #include +#include //#define NAPT_DEBUG 1 #ifdef NAPT_DEBUG diff --git a/src/net/super_stack.cpp b/src/net/super_stack.cpp index b3f876c78f..95171a1b8d 100644 --- a/src/net/super_stack.cpp +++ b/src/net/super_stack.cpp @@ -16,15 +16,14 @@ // limitations under the License. #include -#include +#include #include namespace net { // Specialization for IP4 -template <> -Inet& Super_stack::create(hw::Nic& nic, int N, int sub) +Inet& Super_stack::create(hw::Nic& nic, int N, int sub) { INFO("Network", "Creating stack for %s on %s (MTU=%u)", nic.driver_name(), nic.device_name().c_str(), nic.MTU()); @@ -40,7 +39,7 @@ Inet& Super_stack::create(hw::Nic& nic, int N, int sub) auto inet = [&nic]()->auto { switch(nic.proto()) { case hw::Nic::Proto::ETH: - return std::make_unique(nic); + return std::make_unique(nic); default: throw Super_stack_err{"Nic not supported"}; } @@ -54,8 +53,7 @@ Inet& Super_stack::create(hw::Nic& nic, int N, int sub) } // Specialization for IP4 -template <> -Inet& Super_stack::create(hw::Nic& nic) +Inet& Super_stack::create(hw::Nic& nic) { INFO("Network", "Creating stack for %s on %s (MTU=%u)", nic.driver_name(), nic.device_name().c_str(), nic.MTU()); @@ -63,7 +61,7 @@ Inet& Super_stack::create(hw::Nic& nic) auto inet_ = [&nic]()->auto { switch(nic.proto()) { case hw::Nic::Proto::ETH: - return std::make_unique(nic); + return std::make_unique(nic); default: throw Super_stack_err{"Nic not supported"}; } @@ -84,8 +82,7 @@ Inet& Super_stack::create(hw::Nic& nic) } // Specialization for IP4 -template <> -Inet& Super_stack::get(int N) +Inet& Super_stack::get(int N) { if (N < 0 || N >= (int) hw::Devices::devices().size()) throw Stack_not_found{"No IP4 stack found with index: " + std::to_string(N) + @@ -98,12 +95,11 @@ Inet& Super_stack::get(int N) // create network stack auto& nic = hw::Devices::get(N); - return inet().create(nic, N, 0); + return inet().create(nic, N, 0); } // Specialization for IP4 -template <> -Inet& Super_stack::get(int N, int sub) +Inet& Super_stack::get(int N, int sub) { if (N < 0 || N >= (int) hw::Devices::devices().size()) throw Stack_not_found{"No IP4 stack found with index: " + std::to_string(N) + @@ -123,8 +119,7 @@ Inet& Super_stack::get(int N, int sub) } // Specialization for IP4 -template <> -Inet& Super_stack::get(const std::string& mac) +Inet& Super_stack::get(const std::string& mac) { MAC::Addr link_addr{mac.c_str()}; // Look for the stack with the same NIC @@ -149,7 +144,7 @@ Inet& Super_stack::get(const std::string& mac) throw Stack_not_found{"No NIC found with MAC address " + mac}; // If not found, create - return inet().create(*(*it)); + return inet().create(*(*it)); } Super_stack::Super_stack() @@ -158,7 +153,7 @@ Super_stack::Super_stack() INFO("Network", "No registered network interfaces found"); for (size_t i = 0; i < hw::Devices::devices().size(); i++) { - ip4_stacks_.emplace_back(Stacks{}); + ip4_stacks_.emplace_back(Stacks{}); ip4_stacks_.back()[0] = nullptr; } } diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index 0793a67380..654e7a1d9f 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -20,6 +20,7 @@ #define DEBUG2 #include +#include #include // checksum #include #include // nanos_since_boot (get_ts_value) @@ -493,3 +494,15 @@ void TCP::queue_offer(Connection& conn) } } } + +tcp::Address TCP::address() const noexcept +{ return inet_.ip_addr(); } + +IP4& TCP::network() const +{ return inet_.ip_obj(); } + +bool TCP::is_valid_source(const tcp::Address addr) const noexcept +{ return addr == 0 or inet_.is_valid_source(addr); } + +void TCP::kick() +{ process_writeq(inet_.transmit_queue_available()); } diff --git a/src/plugins/terminal.cpp b/src/plugins/terminal.cpp index 78622fc06a..2026c28fa3 100644 --- a/src/plugins/terminal.cpp +++ b/src/plugins/terminal.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include static int counter = 0; @@ -52,7 +52,7 @@ void store_terminal(liu::Storage& store, const liu::buffer_t*) } void restore_terminal(const int TERM_NET, liu::Restore& restore) { - auto& inet = net::Super_stack::get(TERM_NET); + auto& inet = net::Super_stack::get(TERM_NET); while (!restore.is_end()) { auto conn = restore.as_tcp_connection(inet.tcp()); @@ -75,7 +75,7 @@ static void spawn_terminal() const auto& obj = doc["terminal"]; // terminal network interface const int TERM_NET = obj["iface"].GetInt(); - auto& inet = net::Super_stack::get(TERM_NET); + auto& inet = net::Super_stack::get(TERM_NET); // terminal TCP port const int TERM_PORT = obj["port"].GetUint(); inet.tcp().listen(TERM_PORT, diff --git a/src/plugins/unik.cpp b/src/plugins/unik.cpp index e9c135eef3..83123303f4 100644 --- a/src/plugins/unik.cpp +++ b/src/plugins/unik.cpp @@ -18,7 +18,7 @@ #include #include -#include +#include #include #include @@ -27,7 +27,7 @@ unik::Client::Registered_event unik::Client::on_registered_{nullptr}; /** * UniK instance listener hearbeat / http registration **/ -void unik::Client::register_instance(net::Inet& inet, const net::UDP::port_t port) { +void unik::Client::register_instance(net::Inet& inet, const net::UDP::port_t port) { INFO("Unik client", "Initializing Unik registration service"); INFO("Unik client","Listening for UDP hearbeat on %s:%i", inet.ip_addr().str().c_str(), port); @@ -35,7 +35,7 @@ void unik::Client::register_instance(net::Inet& inet, const net::UDP:: // Set up an UDP port for receiving UniK heartbeat auto& sock = inet.udp().bind(port); - CHECK(net::Inet4::stack<0>().udp().is_bound(sock.local()), "Unik UDP port is bound as expected"); + CHECK(net::Inet::stack<0>().udp().is_bound(sock.local()), "Unik UDP port is bound as expected"); sock.on_read([&sock, &inet] (auto addr, auto port, const char* data, size_t len) { static bool registered_with_unik = false; @@ -106,9 +106,9 @@ void unik::Client::register_instance(net::Inet& inet, const net::UDP:: void unik::Client::register_instance_dhcp() { // Bring up a network device using DHCP - static auto&& inet = net::Inet4::stack<0>(); + static auto&& inet = net::Inet::stack<0>(); - net::Inet4::ifconfig<0>(10.0, [](bool timeout) { + net::Inet::ifconfig<0>(10.0, [](bool timeout) { if(timeout) { INFO("Unik client","DHCP request timed out. Nothing to do."); return; diff --git a/src/posix/tcp_fd.cpp b/src/posix/tcp_fd.cpp index 7c8d38ba5e..9058ff976c 100644 --- a/src/posix/tcp_fd.cpp +++ b/src/posix/tcp_fd.cpp @@ -31,7 +31,7 @@ using namespace net; // return the "currently selected" networking stack static auto& net_stack() { - return Inet4::stack(); + return Inet::stack(); } int TCP_FD::read(void* data, size_t len) diff --git a/src/posix/udp_fd.cpp b/src/posix/udp_fd.cpp index 47eaf11537..fd6afe2a50 100644 --- a/src/posix/udp_fd.cpp +++ b/src/posix/udp_fd.cpp @@ -27,8 +27,8 @@ #endif // return the "currently selected" networking stack -static net::Inet& net_stack() { - return net::Inet4::stack<> (); +static net::Inet& net_stack() { + return net::Inet::stack<> (); } size_t UDP_FD::max_buffer_msgs() const diff --git a/src/util/syslog_facility.cpp b/src/util/syslog_facility.cpp index 82a6ea25f4..7c715f9214 100644 --- a/src/util/syslog_facility.cpp +++ b/src/util/syslog_facility.cpp @@ -15,7 +15,7 @@ // limitations under the License. #include -#include +#include #include // getpid #include @@ -35,7 +35,7 @@ void Syslog_udp::syslog(const std::string& log_message) { void Syslog_udp::open_socket() { if (sock_ == nullptr) - sock_ = &net::Inet4::stack<>().udp().bind(); + sock_ = &net::Inet::stack<>().udp().bind(); } void Syslog_udp::close_socket() { @@ -65,7 +65,7 @@ std::string Syslog_udp::build_message_prefix(const std::string& binary_name) { strftime(timebuf, TIMELEN, "%FT%T.000Z", localtime(&now)); // Third: Hostname ( Preferably: 1. FQDN (RFC1034) 2. Static IP address 3. Hostname 4. Dynamic IP address 5. NILVALUE (-) ) - message += std::string{timebuf} + " " + net::Inet4::stack().ip_addr().str() + " "; + message += std::string{timebuf} + " " + net::Inet::stack().ip_addr().str() + " "; // Fourth: App-name, PROCID (LOG_PID) and MSGID message += std::string(binary_name) + " " + std::to_string(getpid()) + " UDPOUT "; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dc117ec28f..d64cb866bc 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -101,6 +101,7 @@ set(TEST_SOURCES ${TEST}/net/unit/ip4_addr.cpp ${TEST}/net/unit/ip4.cpp ${TEST}/net/unit/ip4_packet_test.cpp + ${TEST}/net/unit/ip6.cpp ${TEST}/net/unit/nat_test.cpp ${TEST}/net/unit/napt_test.cpp ${TEST}/net/unit/packets.cpp @@ -180,13 +181,14 @@ set(OS_SOURCES ${SRC}/net/http/time.cpp ${SRC}/net/http/version.cpp ${SRC}/net/checksum.cpp - ${SRC}/net/inet4.cpp + ${SRC}/net/inet.cpp ${SRC}/net/ip4/arp.cpp ${SRC}/net/ip4/icmp4.cpp ${SRC}/net/ip4/ip4.cpp ${SRC}/net/ip4/reassembly.cpp ${SRC}/net/ip4/udp.cpp ${SRC}/net/ip4/udp_socket.cpp + ${SRC}/net/ip6/ip6.cpp ${SRC}/net/dhcp/dh4client.cpp ${SRC}/net/nat/nat.cpp ${SRC}/net/nat/napt.cpp diff --git a/test/kernel/integration/LiveUpdate/service.cpp b/test/kernel/integration/LiveUpdate/service.cpp index 03548c2943..bc2fa72b75 100644 --- a/test/kernel/integration/LiveUpdate/service.cpp +++ b/test/kernel/integration/LiveUpdate/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include "liu.hpp" @@ -31,7 +31,7 @@ void Service::start() if (OS::is_live_updated() == false) { - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); inet.network_config({10,0,0,49}, {255,255,255,0}, {10,0,0,1}); setup_liveupdate_server(inet, 666, func); diff --git a/test/kernel/integration/modules/mod2/service.cpp b/test/kernel/integration/modules/mod2/service.cpp index 7f11483aa8..2dde55da81 100644 --- a/test/kernel/integration/modules/mod2/service.cpp +++ b/test/kernel/integration/modules/mod2/service.cpp @@ -16,11 +16,11 @@ // limitations under the License. #include -#include +#include void Service::start() { - auto& inet = net::Inet4::stack<0>(); + auto& inet = net::Inet::stack<0>(); inet.network_config({10,0,0,42}, {255,255,255,0}, {10,0,0,1}); diff --git a/test/kernel/integration/term/service.cpp b/test/kernel/integration/term/service.cpp index c570f1f14f..472aa2f21b 100644 --- a/test/kernel/integration/term/service.cpp +++ b/test/kernel/integration/term/service.cpp @@ -17,12 +17,12 @@ #include #include -#include +#include #include void Service::start() { - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); inet.network_config( { 10,0,0,59 }, // IP { 255,255,255,0 }, // Netmask diff --git a/test/linux/router/async_device.hpp b/test/linux/router/async_device.hpp index cddeeb7f6f..6ff81cf9c7 100644 --- a/test/linux/router/async_device.hpp +++ b/test/linux/router/async_device.hpp @@ -18,7 +18,7 @@ #pragma once #include #include -#include +#include #include class Async_device { diff --git a/test/linux/router/nacl.cpp b/test/linux/router/nacl.cpp index dfea196e0f..9f8278bd5f 100644 --- a/test/linux/router/nacl.cpp +++ b/test/linux/router/nacl.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include using namespace net; @@ -9,9 +9,9 @@ std::shared_ptr nacl_ct_obj; void register_plugin_nacl() { INFO("NaCl", "Registering NaCl plugin"); - auto ð0 = Inet4::stack<0>(); + auto ð0 = Inet::stack<0>(); eth0.network_config(IP4::addr{10, 0, 0, 42}, IP4::addr{255, 255, 255, 0}, 0); - auto ð1 = Inet4::stack<1>(); + auto ð1 = Inet::stack<1>(); eth1.network_config(IP4::addr{10, 0, 20, 42}, IP4::addr{255, 255, 255, 0}, 0); return; // Router diff --git a/test/linux/router/service.cpp b/test/linux/router/service.cpp index 850749f528..678b38646c 100644 --- a/test/linux/router/service.cpp +++ b/test/linux/router/service.cpp @@ -17,7 +17,7 @@ void Service::start() hw::Devices::register_device (std::unique_ptr (dev2->get_driver())); // Get the first IP stack configured from config.json - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); inet.network_config({10,0,0,42}, {255,255,255,0}, {10,0,0,1}); extern void register_plugin_nacl(); diff --git a/test/linux/tcp/service.cpp b/test/linux/tcp/service.cpp index 2798f8f862..8fc4277548 100644 --- a/test/linux/tcp/service.cpp +++ b/test/linux/tcp/service.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include #include "../router/async_device.hpp" static const size_t CHUNK_SIZE = 1024 * 1024; @@ -44,8 +44,8 @@ void Service::start() dev2->connect(*dev1); // Create IP stacks on top of the nic's and configure them - auto& inet_server = net::Super_stack::get(0); - auto& inet_client = net::Super_stack::get(1); + auto& inet_server = net::Super_stack::get(0); + auto& inet_client = net::Super_stack::get(1); inet_server.network_config({10,0,0,42}, {255,255,255,0}, {10,0,0,1}); inet_client.network_config({10,0,0,43}, {255,255,255,0}, {10,0,0,1}); diff --git a/test/net/integration/configure/service.cpp b/test/net/integration/configure/service.cpp index e790e01816..74bc489dd3 100644 --- a/test/net/integration/configure/service.cpp +++ b/test/net/integration/configure/service.cpp @@ -16,6 +16,7 @@ // limitations under the License. #include +#include #include #include diff --git a/test/net/integration/dhclient/service.cpp b/test/net/integration/dhclient/service.cpp index fee8157866..056f423d09 100644 --- a/test/net/integration/dhclient/service.cpp +++ b/test/net/integration/dhclient/service.cpp @@ -19,18 +19,18 @@ #include #include -#include +#include using namespace net; void Service::start(const std::string&) { - net::Inet4::ifconfig(10.0, [](bool timeout) { + net::Inet::ifconfig(10.0, [](bool timeout) { if (timeout) panic("DHCP timed out"); INFO("DHCP test", "Got IP from DHCP"); - printf("%s\n", net::Inet4::stack<0>().ip_addr().str().c_str()); + printf("%s\n", net::Inet::stack<0>().ip_addr().str().c_str()); }); INFO("DHCP test", "Waiting for DHCP response\n"); } diff --git a/test/net/integration/dhcpd/service.cpp b/test/net/integration/dhcpd/service.cpp index 5c3c7a6612..758425b295 100644 --- a/test/net/integration/dhcpd/service.cpp +++ b/test/net/integration/dhcpd/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include @@ -29,7 +29,7 @@ void Service::start(const std::string&) // Server - auto& inet = Inet4::ifconfig<0>( + auto& inet = Inet::ifconfig<0>( { 10,0,100,1 }, // IP { 255,255,255,0 }, // Netmask { 10,0,0,1 }, // Gateway @@ -41,37 +41,37 @@ void Service::start(const std::string&) // Client 1 - Inet4::ifconfig<1>(10.0, [] (bool timeout) { + Inet::ifconfig<1>(10.0, [] (bool timeout) { if (timeout) { printf("Client 1 timed out\n"); } else { INFO("DHCP test", "Client 1 got IP from IncludeOS DHCP server"); - printf("%s\n", net::Inet4::stack<1>().ip_addr().str().c_str()); + printf("%s\n", net::Inet::stack<1>().ip_addr().str().c_str()); } }); // Client 2 - Inet4::ifconfig<2>(10.0, [] (bool timeout) { + Inet::ifconfig<2>(10.0, [] (bool timeout) { if (timeout) { printf("Client 2 timed out\n"); } else { INFO("DHCP test", "Client 2 got IP from IncludeOS DHCP server"); - printf("%s\n", net::Inet4::stack<2>().ip_addr().str().c_str()); + printf("%s\n", net::Inet::stack<2>().ip_addr().str().c_str()); } }); // Client 3 - Inet4::ifconfig<3>(10.0, [] (bool timeout) { + Inet::ifconfig<3>(10.0, [] (bool timeout) { if (timeout) { printf("Client 3 timed out\n"); } else { INFO("DHCP test", "Client 3 got IP from IncludeOS DHCP server"); - printf("%s\n", net::Inet4::stack<3>().ip_addr().str().c_str()); + printf("%s\n", net::Inet::stack<3>().ip_addr().str().c_str()); } }); } diff --git a/test/net/integration/dhcpd_dhclient_linux/service.cpp b/test/net/integration/dhcpd_dhclient_linux/service.cpp index f5801588e1..3e2e05cfa1 100644 --- a/test/net/integration/dhcpd_dhclient_linux/service.cpp +++ b/test/net/integration/dhcpd_dhclient_linux/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include @@ -29,7 +29,7 @@ void Service::start(const std::string&) // Server - auto& inet = Inet4::ifconfig<0>( + auto& inet = Inet::ifconfig<0>( { 10,200,0,1 }, // IP { 255,255,0,0 }, // Netmask { 10,0,0,1 }, // Gateway diff --git a/test/net/integration/dns/service.cpp b/test/net/integration/dns/service.cpp index 67aa45ef56..e863443b1c 100644 --- a/test/net/integration/dns/service.cpp +++ b/test/net/integration/dns/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include using namespace net; @@ -42,7 +42,7 @@ void print_success(const std::string& hostname, IP4::addr server, IP4::addr res) void Service::start(const std::string&) { - auto& inet = net::Inet4::stack<0>(); + auto& inet = net::Inet::stack<0>(); inet.network_config( { 10, 0, 0, 48 }, // IP { 255, 255, 255, 0 }, // Netmask diff --git a/test/net/integration/gateway/service.cpp b/test/net/integration/gateway/service.cpp index 22d2037cbe..e653d0445f 100644 --- a/test/net/integration/gateway/service.cpp +++ b/test/net/integration/gateway/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include using namespace net; @@ -29,13 +29,13 @@ void verify() { void Service::start() { - static auto& eth0 = Inet4::stack<0>(); + static auto& eth0 = Inet::stack<0>(); - static auto& eth1 = Inet4::stack<1>(); + static auto& eth1 = Inet::stack<1>(); - static auto& host1 = Inet4::stack<2>(); + static auto& host1 = Inet::stack<2>(); - static auto& host2 = Inet4::stack<3>(); + static auto& host2 = Inet::stack<3>(); INFO("Ping", "host1 => host2 (%s)", host2.ip_addr().to_string().c_str()); host1.icmp().ping(host2.ip_addr(), [](auto reply) { diff --git a/test/net/integration/http/service.cpp b/test/net/integration/http/service.cpp index e54a10f65e..44303b66f2 100644 --- a/test/net/integration/http/service.cpp +++ b/test/net/integration/http/service.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include #include #include @@ -32,7 +32,7 @@ void Service::start(const std::string&) void Service::ready() { - auto& inet = net::Inet4::stack<0>(); // Inet4::stack<0>(); + auto& inet = net::Inet::stack<0>(); // Inet::stack<0>(); inet.network_config( { 10, 0, 0, 46 }, // IP { 255,255,255, 0 }, // Netmask diff --git a/test/net/integration/icmp/service.cpp b/test/net/integration/icmp/service.cpp index be8e26644f..c2e406da87 100644 --- a/test/net/integration/icmp/service.cpp +++ b/test/net/integration/icmp/service.cpp @@ -16,13 +16,13 @@ // limitations under the License. #include -#include +#include using namespace net; void Service::start(const std::string&) { - auto& inet = Inet4::stack<0>(); + auto& inet = Inet::stack<0>(); inet.network_config({ 10, 0, 0, 45 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 1 }, // Gateway diff --git a/test/net/integration/ipv6/CMakeLists.txt b/test/net/integration/ipv6/CMakeLists.txt new file mode 100644 index 0000000000..da27f7519d --- /dev/null +++ b/test/net/integration/ipv6/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 2.8.9) + +# IncludeOS install location +if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) + set(ENV{INCLUDEOS_PREFIX} /usr/local) +endif() + +include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) + +project(test_ipv6) + +# Human-readable name of your service +set(SERVICE_NAME "IncludeOS IPv6 test") + +# Name of your service binary +set(BINARY "test_ipv6") + +# Maximum memory can be hard-coded into the binary +set(MAX_MEM 128) + +# Source files to be linked with OS library parts to form bootable image +set(SOURCES + service.cpp + ) + +# DRIVERS / PLUGINS: + +set(DRIVERS + virtionet + ) + +# include service build script +include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/net/integration/ipv6/service.cpp b/test/net/integration/ipv6/service.cpp new file mode 100644 index 0000000000..2c550f072d --- /dev/null +++ b/test/net/integration/ipv6/service.cpp @@ -0,0 +1,33 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +using namespace net; + +void Service::start(const std::string&) +{ + auto& inet = Inet::stack<0>(); + inet.network_config({10,0,0,42}, {255,255,255,0}, {10,0,0,1}, {0, 0, 0, 0}, + { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd }, // IP6 + { 0, 0, 0, 64 }, // Prefix6 + { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x83e7 } // Gateway6 + ); + printf("Service IPv4 address: %s, IPv6 address: %s\n", + inet.ip_addr().str().c_str(), inet.ip6_addr().str().c_str()); +} diff --git a/test/net/integration/ipv6/vm.json b/test/net/integration/ipv6/vm.json new file mode 100644 index 0000000000..5dea28b28d --- /dev/null +++ b/test/net/integration/ipv6/vm.json @@ -0,0 +1,6 @@ +{ + "image" : "test_ipv6.img", + "net" : [{"device" : "virtio", "mac" : "c0:01:0a:00:00:2a"}], + "mem" : 128, + "intrusive" : "True" +} diff --git a/test/net/integration/nat/service.cpp b/test/net/integration/nat/service.cpp index 70c08013b8..a1aaf23997 100644 --- a/test/net/integration/nat/service.cpp +++ b/test/net/integration/nat/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include @@ -31,12 +31,12 @@ void test_finished() { if (++i == 6) printf("SUCCESS\n"); } -void ip_forward(IP4::IP_packet_ptr pckt, Inet& stack, Conntrack::Entry_ptr) { +void ip_forward(IP4::IP_packet_ptr pckt, Inet& stack, Conntrack::Entry_ptr) { // Packet could have been erroneously moved prior to this call if (not pckt) return; - Inet* route = router->get_first_interface(pckt->ip_dst()); + Inet* route = router->get_first_interface(pckt->ip_dst()); if (not route){ INFO("ip_fwd", "No route found for %s dropping", pckt->ip_dst().to_string().c_str()); @@ -49,19 +49,19 @@ void ip_forward(IP4::IP_packet_ptr pckt, Inet& stack, Conntrack::Entry_ptr) void Service::start() { INFO("NAT Test", "Setting up enviornment to simulate a home router"); - static auto& eth0 = Inet4::ifconfig<0>( + static auto& eth0 = Inet::ifconfig<0>( { 10, 1, 0, 1 }, { 255, 255, 0, 0 }, { 10, 0, 0, 1 }); - static auto& eth1 = Inet4::ifconfig<1>( + static auto& eth1 = Inet::ifconfig<1>( { 192, 1, 0, 1 }, { 255, 255, 255, 0 }, { 10, 0, 0, 1 }); - static auto& laptop1 = Inet4::ifconfig<2>( + static auto& laptop1 = Inet::ifconfig<2>( { 10, 1, 0, 10 }, { 255, 255, 255, 0 }, eth0.ip_addr()); - static auto& internet_host = Inet4::ifconfig<3>( + static auto& internet_host = Inet::ifconfig<3>( { 192, 1, 0, 192 }, { 255, 255, 255, 0 }, eth1.ip_addr()); - static auto& server = Inet4::ifconfig<4>( + static auto& server = Inet::ifconfig<4>( { 10, 1, 10, 20 }, { 255, 255, 255, 0 }, eth0.ip_addr()); @@ -84,12 +84,12 @@ void Service::start() // Setup NAT (Masquerade) natty = std::make_unique(ct); - auto masq = [](IP4::IP_packet_ptr pkt, Inet& stack, Conntrack::Entry_ptr entry)->Filter_verdict + auto masq = [](IP4::IP_packet_ptr pkt, Inet& stack, Conntrack::Entry_ptr entry)->Filter_verdict { natty->masquerade(*pkt, stack, entry); return {std::move(pkt), Filter_verdict_type::ACCEPT}; }; - auto demasq = [](IP4::IP_packet_ptr pkt, Inet& stack, Conntrack::Entry_ptr entry)->Filter_verdict + auto demasq = [](IP4::IP_packet_ptr pkt, Inet& stack, Conntrack::Entry_ptr entry)->Filter_verdict { natty->demasquerade(*pkt, stack, entry); return {std::move(pkt), Filter_verdict_type::ACCEPT}; @@ -133,7 +133,7 @@ void Service::start() static const uint16_t DNAT_PORT{3389}; static const uint16_t DNAT_PORT2{8933}; // DNAT all TCP on dst_port==DNAT_PORT to SERVER - auto dnat_rule = [](IP4::IP_packet_ptr pkt, Inet& stack, Conntrack::Entry_ptr entry)->Filter_verdict + auto dnat_rule = [](IP4::IP_packet_ptr pkt, Inet& stack, Conntrack::Entry_ptr entry)->Filter_verdict { if(not entry) return {std::move(pkt), Filter_verdict_type::DROP}; @@ -150,7 +150,7 @@ void Service::start() return {std::move(pkt), Filter_verdict_type::ACCEPT}; }; // SNAT all packets that comes in return that has been DNAT - auto snat_translate = [](IP4::IP_packet_ptr pkt, Inet& stack, Conntrack::Entry_ptr entry)->Filter_verdict + auto snat_translate = [](IP4::IP_packet_ptr pkt, Inet& stack, Conntrack::Entry_ptr entry)->Filter_verdict { natty->snat(*pkt, entry); return {std::move(pkt), Filter_verdict_type::ACCEPT}; diff --git a/test/net/integration/router/service.cpp b/test/net/integration/router/service.cpp index 0dec58d192..cb80d0756e 100644 --- a/test/net/integration/router/service.cpp +++ b/test/net/integration/router/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include using namespace net; @@ -40,9 +40,9 @@ route_checker(IP4::addr addr) } static void -ip_forward (IP4::IP_packet_ptr pckt, Inet& stack, Conntrack::Entry_ptr) +ip_forward (IP4::IP_packet_ptr pckt, Inet& stack, Conntrack::Entry_ptr) { - Inet* route = router->get_first_interface(pckt->ip_dst()); + Inet* route = router->get_first_interface(pckt->ip_dst()); if (not route){ INFO("ip_fwd", "No route found for %s dropping\n", pckt->ip_dst().to_string().c_str()); @@ -61,14 +61,14 @@ ip_forward (IP4::IP_packet_ptr pckt, Inet& stack, Conntrack::Entry_ptr) void Service::start(const std::string&) { - auto& inet = Inet4::stack<0>(); + auto& inet = Inet::stack<0>(); inet.network_config({ 10, 0, 0, 42 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 1 } ); // Gateway INFO("Router","Interface 1 IP: %s\n", inet.ip_addr().str().c_str()); - auto& inet2 = Inet4::stack<1>(); + auto& inet2 = Inet::stack<1>(); inet2.network_config({ 10, 42, 42, 43 }, // IP { 255, 255, 255, 0 }, // Netmask { 10, 42, 42, 2 } ); // Gateway diff --git a/test/net/integration/tcp/service.cpp b/test/net/integration/tcp/service.cpp index 2ae61580b6..b8bed4aa36 100644 --- a/test/net/integration/tcp/service.cpp +++ b/test/net/integration/tcp/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include #include @@ -58,16 +58,16 @@ void FINISH_TEST() { Timers::oneshot(2 * MSL_TEST + 100ms, [] (Timers::id_t) { INFO("TEST", "Verify release of resources"); - CHECKSERT(Inet4::stack<0>().tcp().active_connections() == 0, + CHECKSERT(Inet::stack<0>().tcp().active_connections() == 0, "No (0) active connections"); INFO("Buffers", "%u avail / %u total", - Inet4::stack<0>().buffers_available(), - Inet4::stack<0>().buffers_total()); + Inet::stack<0>().buffers_available(), + Inet::stack<0>().buffers_total()); // unfortunately we can't know just how many buffers SHOULD be // available, because drivers take some buffers, but there should // be at least half the buffers left - auto total = Inet4::stack<0>().buffers_total(); - CHECKSERT(Inet4::stack<0>().buffers_available() >= total / 2, + auto total = Inet::stack<0>().buffers_total(); + CHECKSERT(Inet::stack<0>().buffers_available() >= total / 2, "Free buffers (%u available)", total); printf("# TEST SUCCESS #\n"); }); @@ -80,13 +80,13 @@ void OUTGOING_TEST_INTERNET(const HostAddress& address) { auto port = address.second; // This needs correct setup to work INFO("TEST", "Outgoing Internet Connection (%s:%u)", address.first.c_str(), address.second); - Inet4::stack<0>().resolve(address.first, + Inet::stack<0>().resolve(address.first, [port](auto ip_address, const Error&) { CHECK(ip_address != 0, "Resolved host"); if(ip_address != 0) { - Inet4::stack<0>().tcp().connect({ip_address, port}) + Inet::stack<0>().tcp().connect({ip_address, port}) ->on_connect([](tcp::Connection_ptr conn) { CHECKSERT(conn != nullptr, "Connected"); @@ -104,7 +104,7 @@ void OUTGOING_TEST_INTERNET(const HostAddress& address) { */ void OUTGOING_TEST(Socket outgoing) { INFO("TEST", "Outgoing Connection (%s)", outgoing.to_string().c_str()); - Inet4::stack<0>().tcp().connect(outgoing, [](tcp::Connection_ptr conn) + Inet::stack<0>().tcp().connect(outgoing, [](tcp::Connection_ptr conn) { CHECKSERT(conn != nullptr, "Connection successfully established."); conn->on_read(small.size(), @@ -151,7 +151,7 @@ void Service::start() for(int i = 0; i < H; i++) huge += TEST_STR; huge += "-end"; - auto& inet = Inet4::stack<0>(); // Inet4::stack<0>(); + auto& inet = Inet::stack<0>(); // Inet::stack<0>(); inet.network_config( { 10, 0, 0, 44 }, // IP { 255,255,255, 0 }, // Netmask @@ -246,7 +246,7 @@ void Service::start() tcp.listen(TEST4).on_connect([](tcp::Connection_ptr conn) { INFO("Test 4","Connection/TCP state"); // There should be at least one connection. - CHECKSERT(Inet4::stack<0>().tcp().active_connections() > 0, "There is (>0) open connection(s)"); + CHECKSERT(Inet::stack<0>().tcp().active_connections() > 0, "There is (>0) open connection(s)"); // Test if connected. CHECKSERT(conn->is_connected(), "Is connected"); // Test if writable. @@ -264,7 +264,7 @@ void Service::start() [conn] (auto) { CHECKSERT(conn->is_state({"TIME-WAIT"}), "State: TIME-WAIT"); INFO("Test 4", "Succeeded. Trigger TEST5"); - OUTGOING_TEST({Inet4::stack().gateway(), TEST5}); + OUTGOING_TEST({Inet::stack().gateway(), TEST5}); }); Timers::oneshot(5s, [] (Timers::id_t) { FINISH_TEST(); }); diff --git a/test/net/integration/transmit/service.cpp b/test/net/integration/transmit/service.cpp index b360a9258b..6f2bfcdc02 100644 --- a/test/net/integration/transmit/service.cpp +++ b/test/net/integration/transmit/service.cpp @@ -18,7 +18,7 @@ //#define DEBUG // Debug supression #include -#include +#include using namespace std; using namespace net; @@ -26,7 +26,7 @@ auto& timer = hw::PIT::instance(); void Service::start(const std::string&) { - static auto& inet = net::Inet4::ifconfig<0>( + static auto& inet = net::Inet::ifconfig<0>( { 10,0,0,45 }, // IP { 255,255,255,0 }, // Netmask { 10,0,0,1 }, // Gateway diff --git a/test/net/integration/udp/service.cpp b/test/net/integration/udp/service.cpp index 3169f17e49..7f5624f1ce 100644 --- a/test/net/integration/udp/service.cpp +++ b/test/net/integration/udp/service.cpp @@ -16,14 +16,14 @@ // limitations under the License. #include -#include +#include #include using namespace net; void Service::start() { - auto& inet = Inet4::stack<0>(); + auto& inet = Inet::stack<0>(); inet.network_config({ 10, 0, 0, 55 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 1 } ); // Gateway diff --git a/test/net/integration/vlan/service.cpp b/test/net/integration/vlan/service.cpp index 3ae680a50d..eaee26a505 100644 --- a/test/net/integration/vlan/service.cpp +++ b/test/net/integration/vlan/service.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include void test_finished() { static int i = 0; @@ -26,16 +26,16 @@ void test_finished() { void Service::start() { - auto& eth0 = net::Super_stack::get(0); - auto& eth1 = net::Super_stack::get(1); + auto& eth0 = net::Super_stack::get(0); + auto& eth1 = net::Super_stack::get(1); net::setup_vlans(); - auto& vlan0_2 = net::Super_stack::get(0,2); - auto& vlan0_42 = net::Super_stack::get(0,42); + auto& vlan0_2 = net::Super_stack::get(0,2); + auto& vlan0_42 = net::Super_stack::get(0,42); - auto& vlan1_2 = net::Super_stack::get(1,2); - auto& vlan1_42 = net::Super_stack::get(1,42); + auto& vlan1_2 = net::Super_stack::get(1,2); + auto& vlan1_42 = net::Super_stack::get(1,42); eth0.tcp().listen(80, [](auto conn) { diff --git a/test/net/integration/websocket/service.cpp b/test/net/integration/websocket/service.cpp index 4a462c81ec..06490a7666 100644 --- a/test/net/integration/websocket/service.cpp +++ b/test/net/integration/websocket/service.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -84,7 +84,7 @@ void websocket_service(net::TCP& tcp, uint16_t port) void Service::start() { - auto& inet = net::Inet4::ifconfig<>(0); + auto& inet = net::Inet::ifconfig<>(0); inet.network_config( { 10, 0, 0, 54 }, // IP { 255,255,255, 0 }, // Netmask diff --git a/test/net/unit/error.cpp b/test/net/unit/error.cpp index 4cc6fb0ea8..9444cbf91e 100644 --- a/test/net/unit/error.cpp +++ b/test/net/unit/error.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include using namespace net; diff --git a/test/net/unit/ip4.cpp b/test/net/unit/ip4.cpp index 90513237aa..afdef6deab 100644 --- a/test/net/unit/ip4.cpp +++ b/test/net/unit/ip4.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include int ip_packets_dropped = 0; int ip_packets_received = 0; @@ -98,7 +98,7 @@ CASE("IP4 is still a thing") SETUP("A pre-wired IP4 instance with custom upstream handlers"){ Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; inet.ip_obj().set_drop_handler(ip_drop); inet.ip_obj().set_udp_handler(ip_rcv_udp); diff --git a/test/net/unit/ip6.cpp b/test/net/unit/ip6.cpp new file mode 100644 index 0000000000..4364d3e184 --- /dev/null +++ b/test/net/unit/ip6.cpp @@ -0,0 +1,47 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#define MYINFO(X,...) INFO("Unit IP6", X, ##__VA_ARGS__) + +using namespace net; +CASE("IP6 packet setters/getters") +{ + INFO("Unit", "IP6 packet setters/getters"); + SETUP("A pre-wired IP6 instance"){ + Nic_mock nic; + Inet inet{nic}; + + SECTION("Inet-created IP packets are initialized correctly") { + auto ip_pckt1 = inet.create_ip6_packet(Protocol::ICMPv4); + EXPECT(ip_pckt1->ip6_version() == 6); + EXPECT(ip_pckt1->is_ipv6()); + EXPECT(ip_pckt1->ip_dscp() == DSCP::CS0); + EXPECT(ip_pckt1->ip_ecn() == ECN::NOT_ECT); + EXPECT(ip_pckt1->flow_label() == 0); + EXPECT(ip_pckt1->payload_length() == 20); + EXPECT(ip_pckt1->next_protocol() == Protocol::ICMPv4); + EXPECT(ip_pckt1->hop_limit() == 0); + EXPECT(ip_pckt1->ip_src() == IP6::ADDR_ANY); + EXPECT(ip_pckt1->ip_dst() == IP6::ADDR_ANY); + } + } +} diff --git a/test/net/unit/napt_test.cpp b/test/net/unit/napt_test.cpp index 6610aa91a3..3e258b007e 100644 --- a/test/net/unit/napt_test.cpp +++ b/test/net/unit/napt_test.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include using namespace net; using namespace net::nat; @@ -161,7 +161,7 @@ CASE("NAPT MASQUERADE") NAPT napt{conntrack}; Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; inet.network_config({10,0,0,40},{255,255,255,0}, 0); const Socket src{{10,0,0,1}, 32222}; diff --git a/test/net/unit/path_mtu_discovery.cpp b/test/net/unit/path_mtu_discovery.cpp index 7f138a0071..b29d1f0f1b 100644 --- a/test/net/unit/path_mtu_discovery.cpp +++ b/test/net/unit/path_mtu_discovery.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include using namespace net; @@ -27,7 +27,7 @@ using namespace net; CASE("Path MTU Discovery is disabled by default and can be enabled") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; EXPECT(not inet.ip_obj().path_mtu_discovery()); inet.set_path_mtu_discovery(true); @@ -37,7 +37,7 @@ CASE("Path MTU Discovery is disabled by default and can be enabled") CASE("An ICMP Too Big message with a next hop MTU value of 45 is invalid") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; inet.set_path_mtu_discovery(true); @@ -64,7 +64,7 @@ CASE("An ICMP Too Big message with a next hop MTU value of 45 is invalid") CASE("An ICMP Too Big message with a next hop MTU value of 68 is valid") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; inet.set_path_mtu_discovery(true); @@ -92,7 +92,7 @@ CASE("An ICMP Too Big message with a next hop MTU value of 68 is valid") CASE("Turning Path MTU Discovery off clears the PMTU cache") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; inet.set_path_mtu_discovery(true); @@ -112,7 +112,7 @@ CASE("Turning Path MTU Discovery off clears the PMTU cache") CASE("No PMTU entry exists for non-existing path") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; inet.set_path_mtu_discovery(true); @@ -122,7 +122,7 @@ CASE("No PMTU entry exists for non-existing path") CASE("A PMTU entry can be removed") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; inet.set_path_mtu_discovery(true); @@ -142,7 +142,7 @@ CASE("A PMTU entry can be removed") CASE("The PMTU cache/map can be flushed") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; inet.set_path_mtu_discovery(true); @@ -172,7 +172,7 @@ CASE("The PMTU cache/map can be flushed") CASE("Path MTU Discovery is enabled by default and can be disabled") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; EXPECT(inet.ip_obj().path_mtu_discovery()); inet.set_path_mtu_discovery(false); @@ -182,7 +182,7 @@ CASE("Path MTU Discovery is enabled by default and can be disabled") CASE("An ICMP Too Big message with a next hop MTU value of 45 is invalid") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; ICMP_error err{icmp4::Type::DEST_UNREACHABLE, (uint8_t) icmp4::code::Dest_unreachable::FRAGMENTATION_NEEDED, 45}; bool too_big = err.is_too_big(); @@ -207,7 +207,7 @@ CASE("An ICMP Too Big message with a next hop MTU value of 45 is invalid") CASE("An ICMP Too Big message with a next hop MTU value of 68 is valid") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; ICMP_error err{icmp4::Type::DEST_UNREACHABLE, (uint8_t) icmp4::code::Dest_unreachable::FRAGMENTATION_NEEDED, 68}; bool too_big = err.is_too_big(); @@ -233,7 +233,7 @@ CASE("An ICMP Too Big message with a next hop MTU value of 68 is valid") CASE("Turning Path MTU Discovery off clears the PMTU cache") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; ICMP_error err{icmp4::Type::DEST_UNREACHABLE, (uint8_t) icmp4::code::Dest_unreachable::FRAGMENTATION_NEEDED, 1400}; bool too_big = err.is_too_big(); @@ -251,7 +251,7 @@ CASE("Turning Path MTU Discovery off clears the PMTU cache") CASE("No PMTU entry exists for non-existing path") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; EXPECT(inet.ip_obj().pmtu(Socket{{10,0,0,45}, 80}) == 0); } @@ -259,7 +259,7 @@ CASE("No PMTU entry exists for non-existing path") CASE("A PMTU entry can be removed") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; ICMP_error err{icmp4::Type::DEST_UNREACHABLE, (uint8_t) icmp4::code::Dest_unreachable::FRAGMENTATION_NEEDED, 1400}; bool too_big = err.is_too_big(); @@ -277,7 +277,7 @@ CASE("A PMTU entry can be removed") CASE("The PMTU cache/map can be flushed") { Nic_mock nic; - Inet4 inet{nic}; + Inet inet{nic}; ICMP_error err1{icmp4::Type::DEST_UNREACHABLE, (uint8_t) icmp4::code::Dest_unreachable::FRAGMENTATION_NEEDED, 1000}; bool too_big1 = err1.is_too_big(); diff --git a/test/net/unit/router_test.cpp b/test/net/unit/router_test.cpp index 8ebd0a6dcd..4ba9871efd 100644 --- a/test/net/unit/router_test.cpp +++ b/test/net/unit/router_test.cpp @@ -24,10 +24,10 @@ using namespace net; // We need discrete inet pointers for the tests -Inet* eth1 = (Inet*) 1; -Inet* eth2 = (Inet*) 2; -Inet* eth3 = (Inet*) 3; -Inet* eth4 = (Inet*) 4; +Inet* eth1 = (Inet*) 1; +Inet* eth2 = (Inet*) 2; +Inet* eth3 = (Inet*) 3; +Inet* eth4 = (Inet*) 4; CASE("net::router: Testing Route functionality") { @@ -154,16 +154,16 @@ CASE("net::router: Testing default gateway route") #include #include -#include +#include CASE("net::router: Actual routing verifying TTL") { Nic_mock nic1; - Inet4 inet1{nic1}; + Inet inet1{nic1}; inet1.network_config({10,0,1,1},{255,255,255,0}, 0); Nic_mock nic2; - Inet4 inet2{nic2}; + Inet inet2{nic2}; inet2.network_config({10,0,2,1},{255,255,255,0}, 0); Router::Routing_table tbl{ @@ -263,7 +263,7 @@ CASE("net::router: Actual routing verifying TTL") CASE("net::router: Calculate Route nexthop") { Nic_mock nic1; - Inet4 inet1{nic1}; + Inet inet1{nic1}; inet1.network_config({10, 0, 2, 1}, {255, 255, 255, 0}, 0); // Net for inet1 without nexthop @@ -284,7 +284,7 @@ CASE("net::router: Calculate Route nexthop") EXPECT(r2.nexthop({10,0,2,20}) == ip4::Addr(10,0,2,20)); // == ip Nic_mock nic2; - Inet4 inet2{nic2}; + Inet inet2{nic2}; inet2.network_config({10, 0, 1, 1}, {255, 255, 255, 0}, 0); // Default route diff --git a/test/net/unit/super_stack.cpp b/test/net/unit/super_stack.cpp index bab581698f..54e62d1c27 100644 --- a/test/net/unit/super_stack.cpp +++ b/test/net/unit/super_stack.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include using namespace net; @@ -39,13 +39,13 @@ CASE("Super stack functionality") EXPECT(Super_stack::inet().ip4_stacks().size() == 3); // Retreiving the first stack creates an interface on the first nic - auto& stack1 = Super_stack::get(0); + auto& stack1 = Super_stack::get(0); EXPECT(&stack1.nic() == nics[0].get()); // Trying to get a stack that do not exists will throw stack_not_found = false; try { - Super_stack::get(3); + Super_stack::get(3); } catch(const Stack_not_found&) { stack_not_found = true; } @@ -55,13 +55,13 @@ CASE("Super stack functionality") const MAC::Addr my_mac{0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; // hehe.. reinterpret_cast(nics[0].get())->mac_ = my_mac; - auto& stack_by_mac = Super_stack::get(my_mac.to_string()); + auto& stack_by_mac = Super_stack::get(my_mac.to_string()); EXPECT(&stack_by_mac.nic() == nics[0].get()); // Throws if mac addr isnt found stack_not_found = false; try { - Super_stack::get("FF:FF:FF:00:00:00"); + Super_stack::get("FF:FF:FF:00:00:00"); } catch(const Stack_not_found&) { stack_not_found = true; } @@ -69,27 +69,27 @@ CASE("Super stack functionality") // Creating substacks works alrite Nic_mock my_nic; - auto& my_sub_stack = Super_stack::inet().create(my_nic, 2, 42); - EXPECT(&my_sub_stack == &Super_stack::get(2,42)); + auto& my_sub_stack = Super_stack::inet().create(my_nic, 2, 42); + EXPECT(&my_sub_stack == &Super_stack::get(2,42)); // Not allowed to create if already occupied tho stack_err = false; try { - Super_stack::inet().create(my_nic, 0, 0); + Super_stack::inet().create(my_nic, 0, 0); } catch(const Super_stack_err&) { stack_err = true; } EXPECT(stack_err == true); // Also possible to create without assigning index, which means it takes the first free one - auto& custom_created_stack = Super_stack::inet().create(my_nic); - EXPECT(&custom_created_stack == &Super_stack::get(1)); + auto& custom_created_stack = Super_stack::inet().create(my_nic); + EXPECT(&custom_created_stack == &Super_stack::get(1)); // Not allowed to create if all indexes are occupied tho - Super_stack::get(2); // occupy the last free one + Super_stack::get(2); // occupy the last free one stack_err = false; try { - Super_stack::inet().create(my_nic); + Super_stack::inet().create(my_nic); } catch(const Super_stack_err&) { stack_err = true; } diff --git a/test/plugin/integration/unik/service.cpp b/test/plugin/integration/unik/service.cpp index ed04a0e90d..700c5e8e1c 100644 --- a/test/plugin/integration/unik/service.cpp +++ b/test/plugin/integration/unik/service.cpp @@ -29,21 +29,21 @@ void Service::start(const std::string&) INFO("Unik test", "SUCCESS"); }); - net::Inet4::ifconfig<0>(5.0, [](auto timeout){ + net::Inet::ifconfig<0>(5.0, [](auto timeout){ CHECK(true, "A service can subscribe to the DHCP event even if Unik did so first"); if (timeout) { INFO("Unik test", "DHCP timed out"); - CHECKSERT(not net::Inet4::stack<0>().udp().is_bound(unik::default_port), "Unik UDP port is free as expected"); + CHECKSERT(not net::Inet::stack<0>().udp().is_bound(unik::default_port), "Unik UDP port is free as expected"); INFO("Unik test", "Manual netwok config"); - net::Inet4::stack<0>().network_config({10,0,0,42},{255,255,255,0},{10,0,0,1},{8,8,8,8}); - unik::Client::register_instance(net::Inet4::stack<0>()); + net::Inet::stack<0>().network_config({10,0,0,42},{255,255,255,0},{10,0,0,1},{8,8,8,8}); + unik::Client::register_instance(net::Inet::stack<0>()); } else { INFO("Unik test", "DHCP OK. We can now use the IP stack"); - CHECK(net::Inet4::stack<0>().udp().is_bound(unik::default_port), "Unik UDP port is bound as expected"); + CHECK(net::Inet::stack<0>().udp().is_bound(unik::default_port), "Unik UDP port is bound as expected"); try { - net::Inet4::stack<0>().udp().bind(unik::default_port); + net::Inet::stack<0>().udp().bind(unik::default_port); } catch(net::UDP::Port_in_use_exception& e){ CHECK(true, "Trying to bound to the Unik port now fails"); INFO("Unik test", "SUCCESS"); diff --git a/test/posix/integration/syslog_plugin/service.cpp b/test/posix/integration/syslog_plugin/service.cpp index 3fde3b2017..5b3dadea4d 100644 --- a/test/posix/integration/syslog_plugin/service.cpp +++ b/test/posix/integration/syslog_plugin/service.cpp @@ -23,7 +23,7 @@ /* For testing IncludeOS */ #include -#include +#include int main() { @@ -32,7 +32,7 @@ int main() /* ------------------------- Testing POSIX syslog ------------------------- */ // DHCP on interface 0 - auto& inet = net::Inet4::stack(); + auto& inet = net::Inet::stack(); // static IP in case DHCP fails inet.network_config({ 10, 0, 0, 47 }, // IP { 255, 255, 255, 0 }, // Netmask diff --git a/test/posix/integration/udp/service.cpp b/test/posix/integration/udp/service.cpp index 8c41f417ab..d10c6ebfb8 100644 --- a/test/posix/integration/udp/service.cpp +++ b/test/posix/integration/udp/service.cpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include const uint16_t PORT = 1042; const uint16_t OUT_PORT = 4242; @@ -30,7 +30,7 @@ const uint16_t BUFSIZE = 2048; int main() { - auto&& inet = net::Inet4::ifconfig({ 10, 0, 0, 50 }, // IP + auto&& inet = net::Inet::ifconfig({ 10, 0, 0, 50 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 3 }); // Gateway diff --git a/test/stress/service.cpp b/test/stress/service.cpp index a635d54f52..1195e6fdc4 100644 --- a/test/stress/service.cpp +++ b/test/stress/service.cpp @@ -16,7 +16,7 @@ // limitations under the License. #include -#include +#include #include #include // rand() #include @@ -76,7 +76,7 @@ void Service::start(const std::string&) for (int i = 0; i < 1000; i++) Timers::oneshot(std::chrono::microseconds(i + 200), [](auto){}); - static auto& inet = net::Inet4::stack<0>(); + static auto& inet = net::Inet::stack<0>(); // Static IP configuration, until we (possibly) get DHCP // @note : Mostly to get a robust demo service that it works with and without DHCP diff --git a/test/util/integration/tar/tar_example/l1_f1/l2/service.cpp b/test/util/integration/tar/tar_example/l1_f1/l2/service.cpp index dd0ff0daf5..94f8d4dda8 100644 --- a/test/util/integration/tar/tar_example/l1_f1/l2/service.cpp +++ b/test/util/integration/tar/tar_example/l1_f1/l2/service.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include using namespace std::chrono; @@ -68,9 +68,9 @@ const std::string NOT_FOUND = "HTTP/1.1 404 Not Found\nConnection: close\n\n"; void Service::start(const std::string&) { // DHCP on interface 0 - auto& inet = net::Inet4::ifconfig(10.0); + auto& inet = net::Inet::ifconfig(10.0); // static IP in case DHCP fails - net::Inet4::ifconfig( + net::Inet::ifconfig( { 10,0,0,42 }, // IP { 255,255,255,0 }, // Netmask { 10,0,0,1 }, // Gateway diff --git a/test/util/integration/tar/tar_example/l1_f1/service.cpp b/test/util/integration/tar/tar_example/l1_f1/service.cpp index dd0ff0daf5..94f8d4dda8 100644 --- a/test/util/integration/tar/tar_example/l1_f1/service.cpp +++ b/test/util/integration/tar/tar_example/l1_f1/service.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include using namespace std::chrono; @@ -68,9 +68,9 @@ const std::string NOT_FOUND = "HTTP/1.1 404 Not Found\nConnection: close\n\n"; void Service::start(const std::string&) { // DHCP on interface 0 - auto& inet = net::Inet4::ifconfig(10.0); + auto& inet = net::Inet::ifconfig(10.0); // static IP in case DHCP fails - net::Inet4::ifconfig( + net::Inet::ifconfig( { 10,0,0,42 }, // IP { 255,255,255,0 }, // Netmask { 10,0,0,1 }, // Gateway diff --git a/test/util/integration/tar_gz/tar_example/l1_f1/l2/service.cpp b/test/util/integration/tar_gz/tar_example/l1_f1/l2/service.cpp index dd0ff0daf5..94f8d4dda8 100644 --- a/test/util/integration/tar_gz/tar_example/l1_f1/l2/service.cpp +++ b/test/util/integration/tar_gz/tar_example/l1_f1/l2/service.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include using namespace std::chrono; @@ -68,9 +68,9 @@ const std::string NOT_FOUND = "HTTP/1.1 404 Not Found\nConnection: close\n\n"; void Service::start(const std::string&) { // DHCP on interface 0 - auto& inet = net::Inet4::ifconfig(10.0); + auto& inet = net::Inet::ifconfig(10.0); // static IP in case DHCP fails - net::Inet4::ifconfig( + net::Inet::ifconfig( { 10,0,0,42 }, // IP { 255,255,255,0 }, // Netmask { 10,0,0,1 }, // Gateway diff --git a/test/util/integration/tar_gz/tar_example/l1_f1/service.cpp b/test/util/integration/tar_gz/tar_example/l1_f1/service.cpp index dd0ff0daf5..94f8d4dda8 100644 --- a/test/util/integration/tar_gz/tar_example/l1_f1/service.cpp +++ b/test/util/integration/tar_gz/tar_example/l1_f1/service.cpp @@ -19,7 +19,7 @@ #include #include -#include +#include #include using namespace std::chrono; @@ -68,9 +68,9 @@ const std::string NOT_FOUND = "HTTP/1.1 404 Not Found\nConnection: close\n\n"; void Service::start(const std::string&) { // DHCP on interface 0 - auto& inet = net::Inet4::ifconfig(10.0); + auto& inet = net::Inet::ifconfig(10.0); // static IP in case DHCP fails - net::Inet4::ifconfig( + net::Inet::ifconfig( { 10,0,0,42 }, // IP { 255,255,255,0 }, // Netmask { 10,0,0,1 }, // Gateway From 9dd275cf40ff01bd283ebe52871d7d0226c40de1 Mon Sep 17 00:00:00 2001 From: nikhilap1 Date: Tue, 24 Apr 2018 10:13:07 +0530 Subject: [PATCH 218/723] Fix build failures after merging from dev --- api/net/inet.hpp | 12 ++++++------ api/posix/syslog_udp_socket.hpp | 6 +++--- src/plugins/syslog.cpp | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/api/net/inet.hpp b/api/net/inet.hpp index 3a959d5ffb..d5023d1e00 100644 --- a/api/net/inet.hpp +++ b/api/net/inet.hpp @@ -351,7 +351,7 @@ namespace net { const Vip4_list virtual_ips() const noexcept { return vip4s_; } - const Vip6_list virtual_ips() const noexcept + const Vip6_list virtual_ip6s() const noexcept { return vip6s_; } /** Check if IP4 address is virtual loopback */ @@ -369,18 +369,18 @@ namespace net { } /** add ip address as virtual loopback */ - void add_vip(ip4::addr a) + void add_vip(IP4::addr a) { if (not is_loopback(a)) { - info("inet", "adding virtual ip address %s", a.to_string().c_str()); + INFO("inet", "adding virtual ip address %s", a.to_string().c_str()); vip4s_.emplace_back(a); } } - void add_vip(ip6::addr a) + void add_vip(IP6::addr a) { if (not is_loopback(a)) { - info("inet", "adding virtual ip6 address %s", a.to_string().c_str()); + INFO("inet", "adding virtual ip6 address %s", a.to_string().c_str()); vip6s_.emplace_back(a); } } @@ -421,7 +421,7 @@ namespace net { if (is_loopback(dest)) return dest; - return ip_addr(); + return ip6_addr(); } bool is_valid_source(IP4::addr src) diff --git a/api/posix/syslog_udp_socket.hpp b/api/posix/syslog_udp_socket.hpp index 1a38ab89df..e744e14b97 100644 --- a/api/posix/syslog_udp_socket.hpp +++ b/api/posix/syslog_udp_socket.hpp @@ -24,7 +24,7 @@ class Syslog_UDP_socket : public Unix_FD_impl { public: - inline Syslog_UDP_socket(net::Inet& net, + inline Syslog_UDP_socket(net::Inet& net, const net::ip4::Addr raddr, const uint16_t rport); inline long connect(const struct sockaddr *, socklen_t) override; @@ -35,13 +35,13 @@ class Syslog_UDP_socket : public Unix_FD_impl { inline ~Syslog_UDP_socket(); private: - net::Inet& stack; + net::Inet& stack; net::UDPSocket* udp; const net::ip4::Addr addr; const uint16_t port; }; -Syslog_UDP_socket::Syslog_UDP_socket(net::Inet& net, +Syslog_UDP_socket::Syslog_UDP_socket(net::Inet& net, const net::ip4::Addr raddr, const uint16_t rport) : stack{net}, udp{nullptr}, addr{raddr}, port{rport} diff --git a/src/plugins/syslog.cpp b/src/plugins/syslog.cpp index 3591c489af..7992a6c6dd 100644 --- a/src/plugins/syslog.cpp +++ b/src/plugins/syslog.cpp @@ -42,7 +42,7 @@ static void mount_print_sock(const std::string& path) fs::mount(path, *syslog_impl, "Syslog Print Unix Backend"); } -static void mount_udp_sock(const std::string& path, net::Inet& stack, +static void mount_udp_sock(const std::string& path, net::Inet& stack, const net::ip4::Addr addr, const uint16_t port) { INFO("Syslog", "Mounting Syslog UDP backend on %s", path.c_str()); @@ -98,7 +98,7 @@ static void syslog_mount() Expects(cfg.HasMember("iface") && "Missing iface (index)"); Expects(cfg.HasMember("address") && "Missing address"); - auto& stack = net::Super_stack::get(cfg["iface"].GetInt()); + auto& stack = net::Super_stack::get(cfg["iface"].GetInt()); const net::ip4::Addr addr{cfg["address"].GetString()}; const uint16_t port = cfg.HasMember("port") ? cfg["port"].GetUint() : default_port; From 2a3bd979d316058094af91dc0e0274908634bdae Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 25 Apr 2018 11:22:20 +0200 Subject: [PATCH 219/723] examples: Add TLS server to LiveUpdate example --- examples/LiveUpdate/CMakeLists.txt | 2 + examples/LiveUpdate/drive/test.key | 52 ++++++++++++++++++++++++++ examples/LiveUpdate/drive/test.pem | 31 +++++++++++++++ examples/LiveUpdate/service.cpp | 60 +++++++++++++++++++++++++++++- 4 files changed, 144 insertions(+), 1 deletion(-) create mode 100644 examples/LiveUpdate/drive/test.key create mode 100644 examples/LiveUpdate/drive/test.pem diff --git a/examples/LiveUpdate/CMakeLists.txt b/examples/LiveUpdate/CMakeLists.txt index b63ce68031..b0f68dd86d 100644 --- a/examples/LiveUpdate/CMakeLists.txt +++ b/examples/LiveUpdate/CMakeLists.txt @@ -34,3 +34,5 @@ set(LIBRARIES # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) + +diskbuilder(drive) diff --git a/examples/LiveUpdate/drive/test.key b/examples/LiveUpdate/drive/test.key new file mode 100644 index 0000000000..0afea2baf1 --- /dev/null +++ b/examples/LiveUpdate/drive/test.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCzNdwPPeuU59Pi +aCSqkmaFYQDWhKZcmX44NaAAzmjNUvNe9u8DEmi+MbmZGYMk5vTNGkPrGM5xZVKs +4urKvtgoWEMrY3uFYyYoqNNpaXrjS5aiBYx2iZDWdDbB0srLOh6liFjfkZVbMNva +gvAb/NV8L6bBWtA54NmA5C0VqfB30dRGKm7CV3wgLyHgmBwHaL31VkahBcTGFFCV +w1QZTNRoTEOhAwb/4rt7YOZkPOOyawCc6B4NUZ3YJ4C7xbJ0t6cv4c/TiofptWDE +RNV2FDkZdh9JLVnJGQtq8Iz9B+0thgKj+JsneYr04KsQ+Ip8EzpqzeqBNFWnzcio +8iBRFxUk9EsQBbSLxKmxe+oDnoznnw1EnTojg8hPvEcDtO9HrAPwYfa7tbj5zkBp +DBPFfYLgZu2D/ARITCVLOB+u4cZo0jIjicX3ubWEXvFVtwdcGm+MUrb7qDMFM0ny +Kw0kvhcVoZHXHUabTc1hCGQVUaI5o4YXpUQv25chtLNdocEN4/Pue89ldjK1ij3c +fwB0z+eShDk9M7H/pr9I7CC1KMxkAbpOiEYj/UfrO+blOgXfc7sH6cOR3ltI79g2 +xW4M3k1gPA24HDaLt1UpNZvgTUhy9MAjespjUMN3XL1WnJhlJC+ydZq2MqJXAA2d +tg2K00zTNK88O58rO9IargmtoeUrSQIDAQABAoICAQClyWmu4AWUV3L4vIdPFOiq +8zYnPcf5WjYeId3HYSwq5lYmwCIUoW8spCuiqqtb9Bz7sRSr5OL2nFmDftjefvbR +O/XHqdyXZUXjz2rk1aPNqhvL/34WGuVWv2P4otzgbP/0+tHc4X1eQzDgUMl32spU +fHCz5yNCp/QO/QeIRxIihobt8ktMlkpKK9AXSiCD6i3xTMNCK2gCJsD2CyE91omZ +gxP9XCOZjVMLrHT2vi2W3M5QWZuTjrGLSeAZ1aZlu5B7B1ePx1Q8rIK8j0E6XzMD +jvcaZ03sb7LUV3zWiAKuXo8Kye4e8p3ONBmNNaBHcDJWo/ARXyzuc7zyLiwfWE9B +uEBJwAC3JNdBdHURP1yeegBgqZEZZfzH8HLZ6dhVQPGbJqqSImKsD2zVhza4GsCh +HqWFLNcrR7xrSuyX0hegvXGIRYjxN9ayFjvIdD5ZIMwpo6FyNJOn/PWOKMRHPqe5 +phTTd/pJrAZ1396XLh0tEVTK2Zl+YAzojfmQH0LIX6Q6loVkpFT7SITfnvSNvnul ++wAGsb+CMvaDicKB3pn8KUtOjBZtOtQJUdY3heCT3CcqMIVunTyQ0fSZFb6ipKKs +/6GjshiC1IBCCBUFWxp6D0JGviFDGZQ1LZpnNatUiWecuU9GbegGVX7RB6OnjR3G +INSyXzKZ9q0f4OThEFGqUQKCAQEA1t6NRL6o8pIhCCSZVuwS1Vx0so0n6WGmvbWa +n4nrx6aAeXO13h9bWUNVNlfPQn9gMHP4iP3IIGm7NP0PL/NpF4AaVnNZeX5SwGFG +LubbmUQBc616ncPdJivZNIFOlqvqxLPxFfG4n/AKeSmeMTEXyxK7OWIipN1vzEiE +uytnkLXSA2Ye4++W+/VwQzTUT3qVhL+RBlM+pOcw1L+wdzRNHcacTSexucgWDMMW +V55SpwUVij8Fvge87am0bBD47Rj80U58cmEFiETw6cCqjmOjqhVHh6VCPdIY3yJd +95oeD7wuvm4CBo+YdhNzOEO2NHtv8YYF+dfGlzg4IKn20MOGqwKCAQEA1YPgvqcu +yOkkI3udtSsM3jvN//u6U9wSC1eE1KoWc8x4ia9Jr1B1Y4mdO+IaIdg6v2x5AzIt +aj5PrnMo+y3nvbndmEBmNPehCbzxUMz+GfNW6zf13Av1sywjALQiwQ2p6vFIFoIX +vMcesd4ZO53CPPisEGOY3qXLzgbEDhyM/zrTPbgNusYvvkzY67PtcGzVlSlwPpJV ++4Hk+ztWrCxNqtl3hjUObBjnuKjGxt+hN2iDfgoVG97yhPzkB8C6AbNTaZqvqg2C +naUx5MOxcll661s7V5bTo4XwEuWWXKj5c2wkO60AHvkjYc7me3i6jlg9YlyPgOi+ +6f0V2u3VkX7l2wKCAQEAovFkigRYFJPSbt1qV2txINIbQSggYzCFhQoJ9wBdiCrw +9KlV+tsmp/uSzEIsz43OwQ/BIwnpUQM9T4K0mLVrNcIUtwiEisjMMk3SLlEtqP3U +aAffm3Jj68WG0vVYRpSa1Y5rvitvygH7v0RbTYygMYTD7FFKWmH+nRlFZrcUs73e +RGuV817Gzc2j06NledxJNMEdVoGcWOtlsYCobs1/yZvK/gujEHL2nbj34XwTy8rk +OdFvJlux3z05sFXyn8K6PnPZldeTnXJCi9Fqxc4z2BCJDQm6wSzpZZUnU1RRhbc8 +b3b3HEia4rf/QWS/8O7Gxo7PS1dhp12f2s1peYk9PwKCAQAxFYIjEhflRAN0zMQy +k9T/ecwfnuT0xlC3nsUAhqFmuYi0TkGoNdzmpwoobBAJ28WVoAApxe0+0VhMCFlR +dPojWYkhqRxV7N9ud6saIiYAHTrMFC9HCNDRAcKCNOcQbm2zfwhNdFa0pSnfRemT +FO9ESP51PhA0jvTNRizn+ZRIUGOjep5dY5YyL0Rm2xQoljx7b+1H1ShDC1dyke+Y +4Q5xylB539SS8R7ECri3m01aiYJBBVxY7eXewKxDRAD+xxTT4CWl+DkguItBxeMT +IJLrbCu2NQwuOWo5TeJFJutBp4ik116BwFBr+b5ugBCTDKH/7LtorRjGfdH6ZFaG +fh+lAoIBABKafHZT9YZPUkvHB5Zs2V5G43PvXv/FM48QBzWGtqLRNnKKtI/JgENV +M1MflSKQLqCRCw/ctHzfPI6zVR7a5AagellJeX/OPJdHN8OiIGHoXtkSHkWETNG3 +JuIN+MgIC/Y7vXtRUt6RwQpkBYq0OlZ4X+2mOYNl4ORglA6OhJnTPzYYxdTeWs/c +00AgkFi8YmITTaAHAG5f609zTz8/LbDie3jwvy0ORiHKfL+B+ihR27zRmNZ/rSbv +9M9m9bKViZoC6Zf/7hN2l7Pjl9IgkUx9Oy3kJSLYqaFpSicN7fGaSkQuw3VDgKvB +Tc4qvMOITdZUl1Das9MqYWp4G8Uk8CA= +-----END PRIVATE KEY----- diff --git a/examples/LiveUpdate/drive/test.pem b/examples/LiveUpdate/drive/test.pem new file mode 100644 index 0000000000..e6ddc43a1a --- /dev/null +++ b/examples/LiveUpdate/drive/test.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFXTCCA0WgAwIBAgIJAMKoGEJAbpuNMA0GCSqGSIb3DQEBDQUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTcwMzA5MTM0NTI2WhcNMjcwMzA3MTM0NTI2WjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAszXcDz3rlOfT4mgkqpJmhWEA1oSmXJl+ODWgAM5ozVLzXvbvAxJovjG5 +mRmDJOb0zRpD6xjOcWVSrOLqyr7YKFhDK2N7hWMmKKjTaWl640uWogWMdomQ1nQ2 +wdLKyzoepYhY35GVWzDb2oLwG/zVfC+mwVrQOeDZgOQtFanwd9HURipuwld8IC8h +4JgcB2i99VZGoQXExhRQlcNUGUzUaExDoQMG/+K7e2DmZDzjsmsAnOgeDVGd2CeA +u8WydLenL+HP04qH6bVgxETVdhQ5GXYfSS1ZyRkLavCM/QftLYYCo/ibJ3mK9OCr +EPiKfBM6as3qgTRVp83IqPIgURcVJPRLEAW0i8SpsXvqA56M558NRJ06I4PIT7xH +A7TvR6wD8GH2u7W4+c5AaQwTxX2C4Gbtg/wESEwlSzgfruHGaNIyI4nF97m1hF7x +VbcHXBpvjFK2+6gzBTNJ8isNJL4XFaGR1x1Gm03NYQhkFVGiOaOGF6VEL9uXIbSz +XaHBDePz7nvPZXYytYo93H8AdM/nkoQ5PTOx/6a/SOwgtSjMZAG6TohGI/1H6zvm +5ToF33O7B+nDkd5bSO/YNsVuDN5NYDwNuBw2i7dVKTWb4E1IcvTAI3rKY1DDd1y9 +VpyYZSQvsnWatjKiVwANnbYNitNM0zSvPDufKzvSGq4JraHlK0kCAwEAAaNQME4w +HQYDVR0OBBYEFAHh+il41QCAviLPiUnybxnw8vDCMB8GA1UdIwQYMBaAFAHh+il4 +1QCAviLPiUnybxnw8vDCMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQENBQADggIB +AIG9NyK5uD5A11VI6gHC8cmfnVszHuaLpuc2y7jyVCyZOt38a0YmKGTUD/u+JKUc +C01uREEi/tqJsm1yE9/kxMMlCGSW9/2kKbbkgU9LdVyVQxr4yVA2aHOf6GDfEioT +O1WXPITZyjBUF5N/t6+LCyClF+0CsDnzWkZkIXEiY5W4Nlqctq1Ef4wMzvCVQ5t6 +RUX8UpxZuraENjtiZlFP8HjQER7/QK+eQl+tI56snAnyU2AcABcUegP1/IphMCDL +iAAGkqBu1wndeeIwUtT9LHuVGWVGEsr1Zpq0NJuVmU5E0i1VRl7whBJQw+PMqFde +6CwI/pMyuZEgh+tXVx8VakI4eSJX8TRPaLeMSI+Sr3CfZxm7Zu6YUEXSLgMrwRqY +atx0oGL8lnYTxlozG+Zmdytge8OwwgtBgBHlmiqbko3ePvlSZTw3gx7tpFBByzX9 +TbgiVKA9P+j5lhdxScls8WjUkr2wXJBXv9WPIK0pUP4UzLJQltamo9lOUJiPgzf9 +v8yPfI4clt4YpRAZw/h1wjhHSYYLmtF6Y8K50NgiFAMuXq5G7q9SlAISt+9oGojA +80MLVkzin8zNuAvGGKnJ72vggKIqIqpdNUyWFH4yuhOtorN+Xam8uPb3axKjVkeP +x2tytEIsJgJY/xggNU1h8myeo8EJLwgXeEVQWMOiE2jI +-----END CERTIFICATE----- diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/service.cpp index 7a90edc681..d0977c6e99 100644 --- a/examples/LiveUpdate/service.cpp +++ b/examples/LiveUpdate/service.cpp @@ -18,7 +18,16 @@ #include #include #include +#include +#include +#include #include "liu.hpp" +static std::deque> strims; + +void save_state(liu::Storage&, const liu::buffer_t*) +{ + +} void Service::start() { @@ -32,5 +41,54 @@ void Service::start() printf(" TCP STATUS:\n%s\n", inet.tcp().status().c_str()); }); - setup_liveupdate_server(inet, 666, nullptr); + const char* tls_cert = "/test.pem"; + const char* tls_key = "/test.key"; + const uint16_t tls_port = 12345; + + fs::memdisk().init_fs( + [] (auto err, auto&) { + assert(!err); + }); + + openssl::init(); + printf("Done, verifying RNG\n"); + openssl::verify_rng(); + printf("Done, creating OpenSSL server\n"); + + auto* ctx = openssl::create_server(tls_cert, tls_key); + printf("Done, listening on TCP port\n"); + + inet.tcp().listen(tls_port, + [ctx] (net::tcp::Connection_ptr conn) { + if (conn != nullptr) + { + auto* stream = new openssl::TLS_stream( + ctx, + std::make_unique(conn) + ); + stream->on_connect( + [stream] (auto&) { + printf("Connected to %s\n", stream->to_string().c_str()); + // --> + strims.push_back(std::unique_ptr (stream)); + // <-- + /* + stream->on_read(8192, [] (auto buf) { + printf("Read: %.*s\n", (int) buf->size(), buf->data()); + }); + */ + }); + stream->on_close( + [stream] () { + delete stream; + }); + } + }); + + setup_liveupdate_server(inet, 666, save_state); +} + +void Service::ready() +{ + printf("Service::ready\n"); } From 8e0f7b8df30375bf85e55346a2598fda136cbb62 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 25 Apr 2018 12:24:39 +0200 Subject: [PATCH 220/723] WIP: LiveUpdate, Streams, serialization --- api/net/botan/tls_server.hpp | 5 +++ api/net/openssl/tls_stream.hpp | 6 ++++ api/net/stream.hpp | 7 ++++ api/net/tcp/connection.hpp | 7 ++++ api/net/ws/websocket.hpp | 4 +++ examples/LiveUpdate/service.cpp | 4 --- lib/LiveUpdate/CMakeLists.txt | 8 +++++ lib/LiveUpdate/liveupdate.hpp | 10 +++--- lib/LiveUpdate/resume.cpp | 4 --- lib/LiveUpdate/serialize_openssl.cpp | 51 ++++++++++++++++++++++++++++ lib/LiveUpdate/serialize_tcp.cpp | 19 +++++++++++ lib/LiveUpdate/storage.hpp | 5 ++- lib/LiveUpdate/update.cpp | 10 ------ 13 files changed, 117 insertions(+), 23 deletions(-) create mode 100644 lib/LiveUpdate/serialize_openssl.cpp diff --git a/api/net/botan/tls_server.hpp b/api/net/botan/tls_server.hpp index e227f0466b..4a6407f45d 100644 --- a/api/net/botan/tls_server.hpp +++ b/api/net/botan/tls_server.hpp @@ -119,6 +119,11 @@ class Server : public Botan::TLS::Callbacks, public net::Stream return m_transport->get_cpuid(); } + /** Not implemented **/ + //size_t serialize_to(void* /*ptr*/) const override { + // throw std::runtime_error("Not implemented"); + //} + protected: void tls_read(buffer_t buf) { diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index cee495dfed..2bc5a2ad51 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -70,6 +70,12 @@ namespace openssl return m_transport->get_cpuid(); } + //size_t serialize_to(void*) const override; + + int my_virtual_member() override { + return 443; + } + private: void tls_read(buffer_t); int tls_perform_stream_write(); diff --git a/api/net/stream.hpp b/api/net/stream.hpp index 02b0f980a3..81ec6bd4ec 100644 --- a/api/net/stream.hpp +++ b/api/net/stream.hpp @@ -26,6 +26,7 @@ #include namespace net { + class Inet4; class Stream; using Stream_ptr = std::unique_ptr; /** @@ -172,6 +173,12 @@ namespace net { **/ virtual int get_cpuid() const noexcept = 0; + //virtual size_t serialize_to(void*) const = 0; + + virtual int my_virtual_member() { + return 0; + } + Stream() = default; virtual ~Stream() {} diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index 17df50b1ef..e6942dd410 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -375,6 +375,13 @@ class Connection { int get_cpuid() const noexcept override; + /* + size_t serialize_to(void* p) const override { + return tcp->serialize_to(p); + }*/ + + virtual int my_virtual_member() override { return 42; } + virtual ~Stream() {} protected: diff --git a/api/net/ws/websocket.hpp b/api/net/ws/websocket.hpp index 14797d3677..47739b607f 100644 --- a/api/net/ws/websocket.hpp +++ b/api/net/ws/websocket.hpp @@ -263,6 +263,10 @@ class WebSocket { max_msg_size = sz; } + size_t serialize_to(void* p) const /*override*/; + /* Create connection from binary data */ + static std::pair deserialize_from(const void*); + WebSocket(net::Stream_ptr, bool); WebSocket(WebSocket&&); ~WebSocket(); diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/service.cpp index d0977c6e99..a351e0bab8 100644 --- a/examples/LiveUpdate/service.cpp +++ b/examples/LiveUpdate/service.cpp @@ -51,9 +51,7 @@ void Service::start() }); openssl::init(); - printf("Done, verifying RNG\n"); openssl::verify_rng(); - printf("Done, creating OpenSSL server\n"); auto* ctx = openssl::create_server(tls_cert, tls_key); printf("Done, listening on TCP port\n"); @@ -72,11 +70,9 @@ void Service::start() // --> strims.push_back(std::unique_ptr (stream)); // <-- - /* stream->on_read(8192, [] (auto buf) { printf("Read: %.*s\n", (int) buf->size(), buf->data()); }); - */ }); stream->on_close( [stream] () { diff --git a/lib/LiveUpdate/CMakeLists.txt b/lib/LiveUpdate/CMakeLists.txt index 3a0642bdd5..dbd211b027 100644 --- a/lib/LiveUpdate/CMakeLists.txt +++ b/lib/LiveUpdate/CMakeLists.txt @@ -9,6 +9,9 @@ include_directories(${MUSL_INCLUDE_DIR}) include_directories(${INCLUDEOS_ROOT}/src/include) include_directories(${INCLUDEOS_ROOT}/api) include_directories(${INCLUDEOS_ROOT}/mod/GSL/) +if (${ARCH} STREQUAL "x86_64") + include_directories(${OPENSSL_DIR}/include) +endif() add_custom_command( OUTPUT hotswap64.bin @@ -17,10 +20,15 @@ add_custom_command( ) add_custom_target(hotswap64 DEPENDS hotswap64.bin) +if(${ARCH} STREQUAL "x86_64") + set(LIU_OPENSSL_FILES "serialize_openssl.cpp") +endif() + # LiveUpdate static library add_library(liveupdate STATIC storage.cpp partition.cpp update.cpp resume.cpp rollback.cpp serialize_tcp.cpp hotswap.cpp hotswap64_blob.asm + ${LIU_OPENSSL_FILES} ) add_dependencies(liveupdate hotswap64) add_dependencies(liveupdate PrecompiledLibraries) diff --git a/lib/LiveUpdate/liveupdate.hpp b/lib/LiveUpdate/liveupdate.hpp index c7ccc33029..3d5578b22d 100644 --- a/lib/LiveUpdate/liveupdate.hpp +++ b/lib/LiveUpdate/liveupdate.hpp @@ -143,6 +143,7 @@ struct Storage inline void add_vector(uid, const std::vector& vector); // store a TCP connection void add_connection(uid, Connection_ptr); + void add_tls_stream(uid, net::Stream&); // markers are used to delineate the end of variable-length structures void put_marker(uid); @@ -172,10 +173,11 @@ struct Restore bool is_end() const noexcept; bool is_int() const noexcept; bool is_marker() const noexcept; - int as_int() const; - std::string as_string() const; - buffer_t as_buffer() const; - Connection_ptr as_tcp_connection(net::TCP&) const; + int as_int() const; + std::string as_string() const; + buffer_t as_buffer() const; + Connection_ptr as_tcp_connection(net::TCP&) const; + net::Stream_ptr as_tls_stream(void* ctx, net::Stream_ptr); template inline const S& as_type() const; diff --git a/lib/LiveUpdate/resume.cpp b/lib/LiveUpdate/resume.cpp index 3862feb450..0ea3730c1f 100644 --- a/lib/LiveUpdate/resume.cpp +++ b/lib/LiveUpdate/resume.cpp @@ -140,10 +140,6 @@ buffer_t Restore::as_buffer() const } throw std::runtime_error("LiveUpdate: Incorrect type " + std::to_string(ent->type)); } -Restore::Connection_ptr Restore::as_tcp_connection(net::TCP& tcp) const -{ - return deserialize_connection(ent->vla, tcp); -} int16_t Restore::get_type() const noexcept { diff --git a/lib/LiveUpdate/serialize_openssl.cpp b/lib/LiveUpdate/serialize_openssl.cpp new file mode 100644 index 0000000000..aea660d6f6 --- /dev/null +++ b/lib/LiveUpdate/serialize_openssl.cpp @@ -0,0 +1,51 @@ +#include +#include +#include "liveupdate.hpp" +#include "storage.hpp" +#include +using namespace net; +typedef net::tcp::Connection_ptr Connection_ptr; + +namespace openssl +{ + /* + size_t TLS_stream::serialize_to(void* loc) const + { + auto* session = SSL_get_session(this->m_ssl); + uint8_t* temp = (uint8_t*) loc; + const int length = i2d_SSL_SESSION(session, &temp); + return length; + }*/ +} + +/// public API /// +namespace liu +{ + void Storage::add_tls_stream(uid id, net::Stream& stream) + { + auto* tls = dynamic_cast (&stream); + hdr.add_struct(TYPE_TLS_STREAM, id, + [tls] (char* location) -> int { + // return size of all the serialized data + return 0;// tls->serialize_to(location); + }); + + } + net::Stream_ptr Restore::as_tls_stream(void* vctx, net::Stream_ptr transport) + { + auto* ctx = (SSL_CTX*) vctx; + // TODO: deserialize SSL session + SSL_SESSION* session = nullptr; + auto* temp = static_cast (this->data()); + auto* rval = d2i_SSL_SESSION(&session, &temp, this->length()); + assert(rval != nullptr); + + int res = SSL_CTX_add_session(ctx, session); + assert(res == 1); + + // TODO: deserialize BIOs + auto stream = std::make_unique (ctx, std::move(transport)); + + return stream; + } +} diff --git a/lib/LiveUpdate/serialize_tcp.cpp b/lib/LiveUpdate/serialize_tcp.cpp index d327d6844d..fade19ee3d 100644 --- a/lib/LiveUpdate/serialize_tcp.cpp +++ b/lib/LiveUpdate/serialize_tcp.cpp @@ -21,6 +21,8 @@ #include #include #include "serialize_tcp.hpp" +#include "liveupdate.hpp" +#include "storage.hpp" #include #include @@ -285,3 +287,20 @@ void serialized_tcp::wakeup_ip_networks() stack->force_start_send_queues(); } } + +/// public API /// +namespace liu +{ + void Storage::add_connection(uid id, Connection_ptr conn) + { + hdr.add_struct(TYPE_TCP, id, + [&conn] (char* location) -> int { + // return size of all the serialized data + return conn->serialize_to(location); + }); + } + Connection_ptr Restore::as_tcp_connection(net::TCP& tcp) const + { + return deserialize_connection(ent->vla, tcp); + } +} diff --git a/lib/LiveUpdate/storage.hpp b/lib/LiveUpdate/storage.hpp index f08374a023..562578df27 100644 --- a/lib/LiveUpdate/storage.hpp +++ b/lib/LiveUpdate/storage.hpp @@ -36,7 +36,10 @@ enum storage_type TYPE_VECTOR = 12, TYPE_STR_VECTOR = 13, - TYPE_TCP = 100, + TYPE_TCP = 100, + TYPE_TCP6 = 101, + TYPE_WEBSOCKET = 105, + TYPE_TLS_STREAM = 106, }; struct segmented_entry diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index c2de1c7d00..c17475c95e 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -328,13 +328,3 @@ void Storage::add_string_vector(uid id, const std::vector& vec) { hdr.add_string_vector(id, vec); } - -#include "serialize_tcp.hpp" -void Storage::add_connection(uid id, Connection_ptr conn) -{ - hdr.add_struct(TYPE_TCP, id, - [&conn] (char* location) -> int { - // return size of all the serialized data - return conn->serialize_to(location); - }); -} From 048b9429384139ceb820fe1a49f9e659f4864f64 Mon Sep 17 00:00:00 2001 From: nikhilap1 Date: Tue, 24 Apr 2018 10:13:07 +0530 Subject: [PATCH 221/723] Fix build failures after merging from dev --- api/net/inet.hpp | 12 ++++++------ api/posix/syslog_udp_socket.hpp | 6 +++--- examples/IRCd/autoconf.cpp | 4 ++-- examples/LiveUpdate/service.cpp | 2 +- examples/TCP_perf/service.cpp | 4 ++-- examples/TLS_server/service.cpp | 2 +- examples/UDP_perf/service.cpp | 4 ++-- examples/acorn/service.cpp | 2 +- examples/demo_linux/service.cpp | 2 +- examples/demo_service/service.cpp | 2 +- examples/http_client/service.cpp | 2 +- examples/router/service.cpp | 4 ++-- examples/scoped_profiler/service.cpp | 2 +- examples/tcp/service.cpp | 2 +- examples/vlan/service.cpp | 4 ++-- src/plugins/syslog.cpp | 4 ++-- test/kernel/integration/LiveUpdate/liu.hpp | 2 +- test/kernel/integration/LiveUpdate/server.hpp | 2 +- 18 files changed, 31 insertions(+), 31 deletions(-) diff --git a/api/net/inet.hpp b/api/net/inet.hpp index 3a959d5ffb..d5023d1e00 100644 --- a/api/net/inet.hpp +++ b/api/net/inet.hpp @@ -351,7 +351,7 @@ namespace net { const Vip4_list virtual_ips() const noexcept { return vip4s_; } - const Vip6_list virtual_ips() const noexcept + const Vip6_list virtual_ip6s() const noexcept { return vip6s_; } /** Check if IP4 address is virtual loopback */ @@ -369,18 +369,18 @@ namespace net { } /** add ip address as virtual loopback */ - void add_vip(ip4::addr a) + void add_vip(IP4::addr a) { if (not is_loopback(a)) { - info("inet", "adding virtual ip address %s", a.to_string().c_str()); + INFO("inet", "adding virtual ip address %s", a.to_string().c_str()); vip4s_.emplace_back(a); } } - void add_vip(ip6::addr a) + void add_vip(IP6::addr a) { if (not is_loopback(a)) { - info("inet", "adding virtual ip6 address %s", a.to_string().c_str()); + INFO("inet", "adding virtual ip6 address %s", a.to_string().c_str()); vip6s_.emplace_back(a); } } @@ -421,7 +421,7 @@ namespace net { if (is_loopback(dest)) return dest; - return ip_addr(); + return ip6_addr(); } bool is_valid_source(IP4::addr src) diff --git a/api/posix/syslog_udp_socket.hpp b/api/posix/syslog_udp_socket.hpp index 1a38ab89df..e744e14b97 100644 --- a/api/posix/syslog_udp_socket.hpp +++ b/api/posix/syslog_udp_socket.hpp @@ -24,7 +24,7 @@ class Syslog_UDP_socket : public Unix_FD_impl { public: - inline Syslog_UDP_socket(net::Inet& net, + inline Syslog_UDP_socket(net::Inet& net, const net::ip4::Addr raddr, const uint16_t rport); inline long connect(const struct sockaddr *, socklen_t) override; @@ -35,13 +35,13 @@ class Syslog_UDP_socket : public Unix_FD_impl { inline ~Syslog_UDP_socket(); private: - net::Inet& stack; + net::Inet& stack; net::UDPSocket* udp; const net::ip4::Addr addr; const uint16_t port; }; -Syslog_UDP_socket::Syslog_UDP_socket(net::Inet& net, +Syslog_UDP_socket::Syslog_UDP_socket(net::Inet& net, const net::ip4::Addr raddr, const uint16_t rport) : stack{net}, udp{nullptr}, addr{raddr}, port{rport} diff --git a/examples/IRCd/autoconf.cpp b/examples/IRCd/autoconf.cpp index 69638cca6d..15bdfd6a69 100644 --- a/examples/IRCd/autoconf.cpp +++ b/examples/IRCd/autoconf.cpp @@ -14,12 +14,12 @@ std::unique_ptr IrcServer::from_config() // client interface const int CLIENT_NET = obj["client_iface"].GetInt(); - auto& clinet = net::Super_stack::get(CLIENT_NET); + auto& clinet = net::Super_stack::get(CLIENT_NET); const int CLIENT_PORT = obj["client_port"].GetUint(); assert(CLIENT_PORT > 0 && CLIENT_PORT < 65536); // server interface const int SERVER_NET = obj["server_iface"].GetInt(); - auto& srvinet = net::Super_stack::get(SERVER_NET); + auto& srvinet = net::Super_stack::get(SERVER_NET); const int SERVER_PORT = obj["server_port"].GetUint(); assert(SERVER_PORT > 0 && SERVER_PORT < 65536); diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/service.cpp index 9f0585d25a..5d11858276 100644 --- a/examples/LiveUpdate/service.cpp +++ b/examples/LiveUpdate/service.cpp @@ -23,7 +23,7 @@ void Service::start() { // Get the first IP stack - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); // Print some useful netstats every 30 secs using namespace std::chrono; diff --git a/examples/TCP_perf/service.cpp b/examples/TCP_perf/service.cpp index 17b02c1895..e19d099953 100644 --- a/examples/TCP_perf/service.cpp +++ b/examples/TCP_perf/service.cpp @@ -100,10 +100,10 @@ void Service::ready() create_network_device(0, "10.0.0.0/24", "10.0.0.1"); // Get the first IP stack configured from config.json - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); inet.network_config({10,0,0,42}, {255,255,255,0}, {10,0,0,1}); #else - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); #endif auto& tcp = inet.tcp(); tcp.set_DACK(dack); // default diff --git a/examples/TLS_server/service.cpp b/examples/TLS_server/service.cpp index 68651e7026..c6bd42fde2 100644 --- a/examples/TLS_server/service.cpp +++ b/examples/TLS_server/service.cpp @@ -27,7 +27,7 @@ void Service::start() { // Get the first IP stack // It should have configuration from config.json - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); // Print some useful netstats every 30 secs Timers::periodic(5s, 30s, diff --git a/examples/UDP_perf/service.cpp b/examples/UDP_perf/service.cpp index ceba251240..f12f350d2f 100644 --- a/examples/UDP_perf/service.cpp +++ b/examples/UDP_perf/service.cpp @@ -132,10 +132,10 @@ void Service::start(const std::string& input) { create_network_device(0, "10.0.0.0/24", "10.0.0.1"); // Get the first IP stack configured from config.json - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); inet.network_config({10,0,0,42}, {255,255,255,0}, {10,0,0,1}); #else - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); #endif auto& udp = inet.udp(); diff --git a/examples/acorn/service.cpp b/examples/acorn/service.cpp index 72bd49e300..99c554b9a9 100644 --- a/examples/acorn/service.cpp +++ b/examples/acorn/service.cpp @@ -164,7 +164,7 @@ static void start_acorn(net::Inet& inet) void Service::start() { - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); if (not inet.is_configured()) { inet.on_config(start_acorn); diff --git a/examples/demo_linux/service.cpp b/examples/demo_linux/service.cpp index d598bfedeb..32f26a9ae2 100644 --- a/examples/demo_linux/service.cpp +++ b/examples/demo_linux/service.cpp @@ -79,7 +79,7 @@ void Service::start() create_network_device(0, "10.0.0.0/24", "10.0.0.1"); // Get the first IP stack configured from config.json - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); inet.network_config({10,0,0,42}, {255,255,255,0}, {10,0,0,1}); // Print some useful netstats every 30 secs diff --git a/examples/demo_service/service.cpp b/examples/demo_service/service.cpp index 5fa19761c8..ee3a65f364 100644 --- a/examples/demo_service/service.cpp +++ b/examples/demo_service/service.cpp @@ -83,7 +83,7 @@ void Service::start() { // Get the first IP stack // It should have configuration from config.json - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); // Print some useful netstats every 30 secs Timers::periodic(5s, 30s, diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index f1bdf31d0b..5ed57fb3a7 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -81,7 +81,7 @@ static void begin_http(net::Inet& inet) void Service::start() { - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); inet.on_config( [] (auto& inet) { diff --git a/examples/router/service.cpp b/examples/router/service.cpp index 5b01b6af35..bbc8366da1 100644 --- a/examples/router/service.cpp +++ b/examples/router/service.cpp @@ -22,8 +22,8 @@ void Service::start() { auto& router = net::get_router(); - auto& eth0 = net::Super_stack::get(0); - auto& eth1 = net::Super_stack::get(1); + auto& eth0 = net::Super_stack::get(0); + auto& eth1 = net::Super_stack::get(1); eth0.set_forward_delg(router.forward_delg()); eth1.set_forward_delg(router.forward_delg()); diff --git a/examples/scoped_profiler/service.cpp b/examples/scoped_profiler/service.cpp index a6da41124e..27e2dbef0e 100644 --- a/examples/scoped_profiler/service.cpp +++ b/examples/scoped_profiler/service.cpp @@ -40,7 +40,7 @@ std::string create_html_response(const std::string& message) void Service::start() { // DHCP on interface 0 - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); // Set up a TCP server on port 80 auto& server = inet.tcp().listen(80); diff --git a/examples/tcp/service.cpp b/examples/tcp/service.cpp index ebba165191..3c0d33f686 100644 --- a/examples/tcp/service.cpp +++ b/examples/tcp/service.cpp @@ -53,7 +53,7 @@ void handle_python_on_read(Connection_ptr client, const std::string& response) { void Service::start() { - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); // Set up a TCP server on port 80 auto& server = inet.tcp().listen(80); diff --git a/examples/vlan/service.cpp b/examples/vlan/service.cpp index 81f781cb11..33cf7f6d58 100644 --- a/examples/vlan/service.cpp +++ b/examples/vlan/service.cpp @@ -22,7 +22,7 @@ void Service::start() { net::setup_vlans(); - auto& eth0 = net::Super_stack::get(0); + auto& eth0 = net::Super_stack::get(0); - auto& vlan0_2 = net::Super_stack::get(0,2); + auto& vlan0_2 = net::Super_stack::get(0,2); } diff --git a/src/plugins/syslog.cpp b/src/plugins/syslog.cpp index 3591c489af..7992a6c6dd 100644 --- a/src/plugins/syslog.cpp +++ b/src/plugins/syslog.cpp @@ -42,7 +42,7 @@ static void mount_print_sock(const std::string& path) fs::mount(path, *syslog_impl, "Syslog Print Unix Backend"); } -static void mount_udp_sock(const std::string& path, net::Inet& stack, +static void mount_udp_sock(const std::string& path, net::Inet& stack, const net::ip4::Addr addr, const uint16_t port) { INFO("Syslog", "Mounting Syslog UDP backend on %s", path.c_str()); @@ -98,7 +98,7 @@ static void syslog_mount() Expects(cfg.HasMember("iface") && "Missing iface (index)"); Expects(cfg.HasMember("address") && "Missing address"); - auto& stack = net::Super_stack::get(cfg["iface"].GetInt()); + auto& stack = net::Super_stack::get(cfg["iface"].GetInt()); const net::ip4::Addr addr{cfg["address"].GetString()}; const uint16_t port = cfg.HasMember("port") ? cfg["port"].GetUint() : default_port; diff --git a/test/kernel/integration/LiveUpdate/liu.hpp b/test/kernel/integration/LiveUpdate/liu.hpp index 689bf82aa6..4519ce39be 100644 --- a/test/kernel/integration/LiveUpdate/liu.hpp +++ b/test/kernel/integration/LiveUpdate/liu.hpp @@ -19,7 +19,7 @@ #include #include "server.hpp" -void setup_liveupdate_server(net::Inet& inet, +void setup_liveupdate_server(net::Inet& inet, const uint16_t PORT, liu::LiveUpdate::storage_func func) { diff --git a/test/kernel/integration/LiveUpdate/server.hpp b/test/kernel/integration/LiveUpdate/server.hpp index 1d2c36a6d5..5ffbb641d0 100644 --- a/test/kernel/integration/LiveUpdate/server.hpp +++ b/test/kernel/integration/LiveUpdate/server.hpp @@ -7,7 +7,7 @@ #include static inline -void server(net::Inet& inet, +void server(net::Inet& inet, const uint16_t port, delegate callback) { From 3c4aae26b27e546a0aecc567cd48368e4084c3e2 Mon Sep 17 00:00:00 2001 From: nikhilap1 Date: Wed, 25 Apr 2018 18:43:17 +0530 Subject: [PATCH 222/723] Address review comments --- api/net/inet.hpp | 34 +++++++++---------- api/net/vlan | 1 + examples/IRCd/ircd/ircd.hpp | 2 +- examples/LiveUpdate/liu.hpp | 2 +- examples/LiveUpdate/server.hpp | 2 +- examples/UDP_perf/service.cpp | 2 +- examples/acorn/service.cpp | 2 +- examples/http_client/service.cpp | 2 +- .../components/dashboard/components/tcp.hpp | 1 + 9 files changed, 25 insertions(+), 23 deletions(-) diff --git a/api/net/inet.hpp b/api/net/inet.hpp index d5023d1e00..e7f8731b99 100644 --- a/api/net/inet.hpp +++ b/api/net/inet.hpp @@ -66,37 +66,37 @@ namespace net { using Vip4_list = std::vector; using Vip6_list = std::vector; - std::string ifname() + std::string ifname() const { return nic_.device_name(); } - MAC::Addr link_addr() + MAC::Addr link_addr() const { return nic_.mac(); } - hw::Nic& nic() + hw::Nic& nic() const { return nic_; } - IP4::addr ip_addr() + IP4::addr ip_addr() const { return ip4_addr_; } - IP4::addr netmask() + IP4::addr netmask() const { return netmask_; } - IP4::addr gateway() + IP4::addr gateway() const { return gateway_; } - IP4::addr dns_addr() + IP4::addr dns_addr() const { return dns_server_; } - IP4::addr broadcast_addr() + IP4::addr broadcast_addr() const { return ip4_addr_ | ( ~ netmask_); } - IP6::addr ip6_addr() + IP6::addr ip6_addr() const { return ip6_addr_; } - IP6::addr netmask6() + IP6::addr netmask6() const { return ip6_prefix_; } - IP6::addr gateway6() + IP6::addr gateway6() const { return ip6_gateway_; } void cache_link_addr(IP4::addr ip, MAC::Addr mac); @@ -211,7 +211,7 @@ namespace net { { return IP_packet_factory{this, &Inet::create_ip_packet}; } /** MTU retreived from Nic on construction */ - uint16_t MTU() + uint16_t MTU () const { return MTU_; } /** @@ -230,7 +230,7 @@ namespace net { void set_domain_name(std::string domain_name) { this->domain_name_ = std::move(domain_name); } - const std::string& domain_name() + const std::string& domain_name() const { return this->domain_name_; } void set_gateway(IP4::addr gateway) @@ -253,12 +253,12 @@ namespace net { */ void negotiate_dhcp(double timeout = 10.0, dhcp_timeout_func = nullptr); - bool is_configured() + bool is_configured() const { return ip4_addr_ != 0; } - bool is_configured_v6() + bool is_configured_v6() const { return ip6_addr_ != IP6::ADDR_ANY; } @@ -284,7 +284,7 @@ namespace net { IP6::addr prefix6 = IP6::ADDR_ANY, IP6::addr gateway6 = IP6::ADDR_ANY); - virtual void + void reset_config() { this->ip4_addr_ = IP4::ADDR_ANY; @@ -296,7 +296,7 @@ namespace net { } // register a callback for receiving signal on free packet-buffers - virtual void + void on_transmit_queue_available(transmit_avail_delg del) { tqa.push_back(del); } diff --git a/api/net/vlan b/api/net/vlan index 8b12f76a71..2371a41c26 100644 --- a/api/net/vlan +++ b/api/net/vlan @@ -21,6 +21,7 @@ #include #include +#include #include #include diff --git a/examples/IRCd/ircd/ircd.hpp b/examples/IRCd/ircd/ircd.hpp index 980c216712..ec7b45577f 100644 --- a/examples/IRCd/ircd/ircd.hpp +++ b/examples/IRCd/ircd/ircd.hpp @@ -33,7 +33,7 @@ struct RemoteServer { class IrcServer { public: using Connection = net::tcp::Connection_ptr; - using Network = net::Inet; + using Network = net::Inet; typedef std::function motd_func_t; IrcServer( diff --git a/examples/LiveUpdate/liu.hpp b/examples/LiveUpdate/liu.hpp index 0d8b18a3fb..1c4e23369c 100644 --- a/examples/LiveUpdate/liu.hpp +++ b/examples/LiveUpdate/liu.hpp @@ -19,7 +19,7 @@ #include #include "server.hpp" -void setup_liveupdate_server(net::Inet& inet, +void setup_liveupdate_server(net::Inet& inet, const uint16_t PORT, liu::LiveUpdate::storage_func func) { diff --git a/examples/LiveUpdate/server.hpp b/examples/LiveUpdate/server.hpp index 4a440f34b3..592e6d16ec 100644 --- a/examples/LiveUpdate/server.hpp +++ b/examples/LiveUpdate/server.hpp @@ -7,7 +7,7 @@ #include static inline -void server(net::Inet& inet, +void server(net::Inet& inet, const uint16_t port, delegate callback) { diff --git a/examples/UDP_perf/service.cpp b/examples/UDP_perf/service.cpp index f12f350d2f..0bca84632c 100644 --- a/examples/UDP_perf/service.cpp +++ b/examples/UDP_perf/service.cpp @@ -116,7 +116,7 @@ void send_cb() { data_len += SEND_BUF_LEN; } -void send_data(net::UDPSocket& client, net::Inet& inet) { +void send_data(net::UDPSocket& client, net::Inet& inet) { for (size_t i = 0; i < PACKETS_PER_INTERVAL; i++) { const char c = 'A' + (i % 26); std::string buff(SEND_BUF_LEN, c); diff --git a/examples/acorn/service.cpp b/examples/acorn/service.cpp index 99c554b9a9..828bfff0d9 100644 --- a/examples/acorn/service.cpp +++ b/examples/acorn/service.cpp @@ -36,7 +36,7 @@ static fs::Disk_ptr disk; #include #include -static void start_acorn(net::Inet& inet) +static void start_acorn(net::Inet& inet) { /** SETUP LOGGER */ const int LOGBUFFER_LEN = 1024*16; diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index 5ed57fb3a7..f710de2250 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -39,7 +39,7 @@ static SSL_CTX* init_ssl_context() #include #include -static void begin_http(net::Inet& inet) +static void begin_http(net::Inet& inet) { using namespace http; diff --git a/lib/mana/include/mana/components/dashboard/components/tcp.hpp b/lib/mana/include/mana/components/dashboard/components/tcp.hpp index f58a3da5cd..1f6764c29e 100644 --- a/lib/mana/include/mana/components/dashboard/components/tcp.hpp +++ b/lib/mana/include/mana/components/dashboard/components/tcp.hpp @@ -21,6 +21,7 @@ #include "../component.hpp" +#include #include namespace mana { From e04f3e2d261170e34490408082d8cdaf015e3b96 Mon Sep 17 00:00:00 2001 From: nikhilap1 Date: Wed, 25 Apr 2018 23:03:56 +0530 Subject: [PATCH 223/723] NaCl change to reflect the new Inet --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index 0227704f0b..874d77668f 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit 0227704f0b5b61906e36e6c41f27b8b6b1157258 +Subproject commit 874d77668f67e1b6bc3f003fbd969243a25bcb07 From 38466b951ac891c7730012f916c5ef35e38017c3 Mon Sep 17 00:00:00 2001 From: nikhilap1 Date: Wed, 25 Apr 2018 23:53:05 +0530 Subject: [PATCH 224/723] Fix posix test failure --- test/posix/integration/tcp/service.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/posix/integration/tcp/service.cpp b/test/posix/integration/tcp/service.cpp index face523028..efdafd7fe4 100644 --- a/test/posix/integration/tcp/service.cpp +++ b/test/posix/integration/tcp/service.cpp @@ -23,7 +23,7 @@ #include #include #include -#include +#include const uint16_t PORT = 1042; const uint16_t OUT_PORT = 4242; @@ -31,7 +31,7 @@ const uint16_t BUFSIZE = 2048; int main() { - auto&& inet = net::Inet4::ifconfig({ 10, 0, 0, 51 }, // IP + auto&& inet = net::Inet::ifconfig({ 10, 0, 0, 51 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 4 }); // Gateway From d3d6e209fc302c9c64b38de70d526209be037532 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 26 Apr 2018 13:28:33 +0200 Subject: [PATCH 225/723] LiveUpdate: Add OpenSSL TLS stream storage support, update LiveUpdate example --- api/net/botan/tls_server.hpp | 10 +++- api/net/openssl/tls_stream.hpp | 18 ++++-- api/net/stream.hpp | 11 ++-- api/net/tcp/connection.hpp | 85 +++++++++++++++------------- examples/LiveUpdate/service.cpp | 57 ++++++++++++++++--- lib/LiveUpdate/liveupdate.hpp | 1 + lib/LiveUpdate/serialize_openssl.cpp | 21 ++++--- lib/LiveUpdate/serialize_tcp.cpp | 4 ++ src/net/tcp/connection.cpp | 2 +- 9 files changed, 140 insertions(+), 69 deletions(-) diff --git a/api/net/botan/tls_server.hpp b/api/net/botan/tls_server.hpp index 4a6407f45d..8d14cc9218 100644 --- a/api/net/botan/tls_server.hpp +++ b/api/net/botan/tls_server.hpp @@ -119,10 +119,14 @@ class Server : public Botan::TLS::Callbacks, public net::Stream return m_transport->get_cpuid(); } + Stream* transport() noexcept override { + return m_transport.get(); + } + /** Not implemented **/ - //size_t serialize_to(void* /*ptr*/) const override { - // throw std::runtime_error("Not implemented"); - //} + size_t serialize_to(void* /*ptr*/) const override { + throw std::runtime_error("Not implemented"); + } protected: void tls_read(buffer_t buf) diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index 2bc5a2ad51..7805780728 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -17,7 +17,8 @@ namespace openssl { using Stream_ptr = net::Stream_ptr; - TLS_stream(SSL_CTX* ctx, Stream_ptr t, bool outgoing = false); + TLS_stream(SSL_CTX* ctx, Stream_ptr, bool outgoing = false); + TLS_stream(Stream_ptr, SSL* ctx, BIO*, BIO*); virtual ~TLS_stream(); void write(buffer_t buffer) override; @@ -70,12 +71,12 @@ namespace openssl return m_transport->get_cpuid(); } - //size_t serialize_to(void*) const override; - - int my_virtual_member() override { - return 443; + Stream* transport() noexcept override { + return m_transport.get(); } + size_t serialize_to(void*) const override; + private: void tls_read(buffer_t); int tls_perform_stream_write(); @@ -127,6 +128,13 @@ namespace openssl if (this->tls_perform_handshake() < 0) return; } } + inline TLS_stream::TLS_stream(Stream_ptr t, SSL* ssl, BIO* rd, BIO* wr) + : m_transport(std::move(t)), m_ssl(ssl), m_bio_rd(rd), m_bio_wr(wr) + { + // always-on callbacks + m_transport->on_read(8192, {this, &TLS_stream::tls_read}); + m_transport->on_close({this, &TLS_stream::close_callback_once}); + } inline TLS_stream::~TLS_stream() { SSL_free(this->m_ssl); diff --git a/api/net/stream.hpp b/api/net/stream.hpp index 81ec6bd4ec..473acb57d5 100644 --- a/api/net/stream.hpp +++ b/api/net/stream.hpp @@ -173,11 +173,14 @@ namespace net { **/ virtual int get_cpuid() const noexcept = 0; - //virtual size_t serialize_to(void*) const = 0; + /** + * Returns the underlying transport, or nullptr if bottom. + * If no transport present, most likely its a TCP stream, in which + * case you can dynamic_cast and call tcp() to get the connection + **/ + virtual Stream* transport() noexcept = 0; - virtual int my_virtual_member() { - return 0; - } + virtual size_t serialize_to(void*) const = 0; Stream() = default; virtual ~Stream() {} diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index e6942dd410..4b3fe51e71 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -221,10 +221,10 @@ class Connection { * @param[in] conn The connection */ Stream(Connection_ptr conn) - : tcp{std::move(conn)} + : m_tcp{std::move(conn)} { // stream for a nullptr makes no sense - Expects(tcp != nullptr); + Expects(m_tcp != nullptr); } /** @@ -232,9 +232,9 @@ class Connection { * * @param[in] cb The connect callback */ - virtual void on_connect(ConnectCallback cb) override + void on_connect(ConnectCallback cb) override { - tcp->on_connect(Connection::ConnectCallback::make_packed( + m_tcp->on_connect(Connection::ConnectCallback::make_packed( [this, cb] (Connection_ptr conn) { if(conn) cb(*this); })); } @@ -245,24 +245,24 @@ class Connection { * @param[in] n The size of the receive buffer * @param[in] cb The read callback */ - virtual void on_read(size_t n, ReadCallback cb) override - { tcp->on_read(n, cb); } + void on_read(size_t n, ReadCallback cb) override + { m_tcp->on_read(n, cb); } /** * @brief Event for when the Stream is being closed. * * @param[in] cb The close callback */ - virtual void on_close(CloseCallback cb) override - { tcp->on_close(cb); } + void on_close(CloseCallback cb) override + { m_tcp->on_close(cb); } /** * @brief Event for when data has been written. * * @param[in] cb The write callback */ - virtual void on_write(WriteCallback cb) override - { tcp->on_write(cb); } + void on_write(WriteCallback cb) override + { m_tcp->on_write(cb); } /** * @brief Async write of a data with a length. @@ -270,8 +270,8 @@ class Connection { * @param[in] buf data * @param[in] n length */ - virtual void write(const void* buf, size_t n) override - { tcp->write(buf, n); } + void write(const void* buf, size_t n) override + { m_tcp->write(buf, n); } /** * @brief Async write of a shared buffer with a length. @@ -279,8 +279,8 @@ class Connection { * @param[in] buffer shared buffer * @param[in] n length */ - virtual void write(buffer_t buffer) override - { tcp->write(buffer); } + void write(buffer_t buffer) override + { m_tcp->write(buffer); } /** * @brief Async write of a string. @@ -288,26 +288,26 @@ class Connection { * * @param[in] str The string */ - virtual void write(const std::string& str) override + void write(const std::string& str) override { write(str.data(), str.size()); } /** * @brief Closes the stream. */ - virtual void close() override - { tcp->close(); } + void close() override + { m_tcp->close(); } /** * @brief Aborts (terminates) the stream. */ - virtual void abort() override - { tcp->abort(); } + void abort() override + { m_tcp->abort(); } /** * @brief Resets all callbacks. */ - virtual void reset_callbacks() override - { tcp->reset_callbacks(); } + void reset_callbacks() override + { m_tcp->reset_callbacks(); } /** * @brief Returns the streams local socket. @@ -315,7 +315,7 @@ class Connection { * @return A TCP Socket */ Socket local() const override - { return tcp->local(); } + { return m_tcp->local(); } /** * @brief Returns the streams remote socket. @@ -323,69 +323,74 @@ class Connection { * @return A TCP Socket */ Socket remote() const override - { return tcp->remote(); } + { return m_tcp->remote(); } /** * @brief Returns a string representation of the stream. * * @return String representation of the stream. */ - virtual std::string to_string() const override - { return tcp->to_string(); } + std::string to_string() const override + { return m_tcp->to_string(); } /** * @brief Determines if connected (established). * * @return True if connected, False otherwise. */ - virtual bool is_connected() const noexcept override - { return tcp->is_connected(); } + bool is_connected() const noexcept override + { return m_tcp->is_connected(); } /** * @brief Determines if writable. (write is allowed) * * @return True if writable, False otherwise. */ - virtual bool is_writable() const noexcept override - { return tcp->is_writable(); } + bool is_writable() const noexcept override + { return m_tcp->is_writable(); } /** * @brief Determines if readable. (data can be received) * * @return True if readable, False otherwise. */ - virtual bool is_readable() const noexcept override - { return tcp->is_readable(); } + bool is_readable() const noexcept override + { return m_tcp->is_readable(); } /** * @brief Determines if closing. * * @return True if closing, False otherwise. */ - virtual bool is_closing() const noexcept override - { return tcp->is_closing(); } + bool is_closing() const noexcept override + { return m_tcp->is_closing(); } /** * @brief Determines if closed. * * @return True if closed, False otherwise. */ - virtual bool is_closed() const noexcept override - { return tcp->is_closed(); }; + bool is_closed() const noexcept override + { return m_tcp->is_closed(); }; int get_cpuid() const noexcept override; - /* size_t serialize_to(void* p) const override { - return tcp->serialize_to(p); - }*/ + return m_tcp->serialize_to(p); + } - virtual int my_virtual_member() override { return 42; } + Stream* transport() noexcept override { + return nullptr; + } virtual ~Stream() {} + Connection_ptr tcp() { + return this->m_tcp; + } + protected: - Connection_ptr tcp; + Connection_ptr m_tcp; }; // < class Connection::Stream diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/service.cpp index a351e0bab8..6f3270f52c 100644 --- a/examples/LiveUpdate/service.cpp +++ b/examples/LiveUpdate/service.cpp @@ -22,11 +22,53 @@ #include #include #include "liu.hpp" -static std::deque> strims; +static std::deque strims; +static SSL_CTX* g_ctx = nullptr; -void save_state(liu::Storage&, const liu::buffer_t*) +static void setup_callbacks(net::Stream& stream) { + stream.on_read(8192, + [] (auto buf) { + printf("Read: %.*s\n", (int) buf->size(), buf->data()); + }); + stream.on_close( + [&stream] () { + printf("Stream to %s closed\n", stream.to_string().c_str()); + }); +} + +static void save_state(liu::Storage& store, const liu::buffer_t*) +{ + printf("Save state called\n"); + for (auto& strim : strims) + { + auto* tls = (openssl::TLS_stream*) strim.get(); + auto* tcp = dynamic_cast (tls->transport()); + store.add_connection(1, tcp->tcp()); + + store.add_tls_stream(2, *tls); + } + store.put_marker(3); +} +static void resume_state(liu::Restore& thing) +{ + printf("Resume state called\n"); + auto& inet = net::Super_stack::get(0); + while (not thing.is_marker()) + { + // restore tcp stream from storage + auto tcp_stream = thing.as_tcp_stream(inet.tcp()); + thing.go_next(); + // create OpenSSL stream using TCP stream + auto tls = thing.as_tls_stream(g_ctx, std::move(tcp_stream)); + thing.go_next(); + printf("Restored stream to %s\n", tls->to_string().c_str()); + // restore callbacks + setup_callbacks(*tls); + // store stream + strims.push_back(std::move(tls)); + } } void Service::start() @@ -53,15 +95,15 @@ void Service::start() openssl::init(); openssl::verify_rng(); - auto* ctx = openssl::create_server(tls_cert, tls_key); + g_ctx = openssl::create_server(tls_cert, tls_key); printf("Done, listening on TCP port\n"); inet.tcp().listen(tls_port, - [ctx] (net::tcp::Connection_ptr conn) { + [] (net::tcp::Connection_ptr conn) { if (conn != nullptr) { auto* stream = new openssl::TLS_stream( - ctx, + g_ctx, std::make_unique(conn) ); stream->on_connect( @@ -70,9 +112,7 @@ void Service::start() // --> strims.push_back(std::unique_ptr (stream)); // <-- - stream->on_read(8192, [] (auto buf) { - printf("Read: %.*s\n", (int) buf->size(), buf->data()); - }); + setup_callbacks(*stream); }); stream->on_close( [stream] () { @@ -82,6 +122,7 @@ void Service::start() }); setup_liveupdate_server(inet, 666, save_state); + liu::LiveUpdate::resume("test", resume_state); } void Service::ready() diff --git a/lib/LiveUpdate/liveupdate.hpp b/lib/LiveUpdate/liveupdate.hpp index 3d5578b22d..4c1edd0921 100644 --- a/lib/LiveUpdate/liveupdate.hpp +++ b/lib/LiveUpdate/liveupdate.hpp @@ -177,6 +177,7 @@ struct Restore std::string as_string() const; buffer_t as_buffer() const; Connection_ptr as_tcp_connection(net::TCP&) const; + net::Stream_ptr as_tcp_stream (net::TCP&) const; net::Stream_ptr as_tls_stream(void* ctx, net::Stream_ptr); template diff --git a/lib/LiveUpdate/serialize_openssl.cpp b/lib/LiveUpdate/serialize_openssl.cpp index aea660d6f6..e541051a14 100644 --- a/lib/LiveUpdate/serialize_openssl.cpp +++ b/lib/LiveUpdate/serialize_openssl.cpp @@ -8,14 +8,13 @@ typedef net::tcp::Connection_ptr Connection_ptr; namespace openssl { - /* size_t TLS_stream::serialize_to(void* loc) const { auto* session = SSL_get_session(this->m_ssl); uint8_t* temp = (uint8_t*) loc; const int length = i2d_SSL_SESSION(session, &temp); return length; - }*/ + } } /// public API /// @@ -27,13 +26,21 @@ namespace liu hdr.add_struct(TYPE_TLS_STREAM, id, [tls] (char* location) -> int { // return size of all the serialized data - return 0;// tls->serialize_to(location); + return tls->serialize_to(location); }); - } net::Stream_ptr Restore::as_tls_stream(void* vctx, net::Stream_ptr transport) { auto* ctx = (SSL_CTX*) vctx; + + SSL* ssl = SSL_new(ctx); + // TODO: deserialize BIOs + BIO* rd = BIO_new(BIO_s_mem()); + BIO* wr = BIO_new(BIO_s_mem()); + assert(ERR_get_error() == 0 && "Initializing BIOs"); + + SSL_set_bio(ssl, rd, wr); + // TODO: deserialize SSL session SSL_SESSION* session = nullptr; auto* temp = static_cast (this->data()); @@ -42,10 +49,8 @@ namespace liu int res = SSL_CTX_add_session(ctx, session); assert(res == 1); + SSL_set_session(ssl, session); - // TODO: deserialize BIOs - auto stream = std::make_unique (ctx, std::move(transport)); - - return stream; + return std::make_unique (std::move(transport), ssl, rd, wr); } } diff --git a/lib/LiveUpdate/serialize_tcp.cpp b/lib/LiveUpdate/serialize_tcp.cpp index fade19ee3d..66709e3238 100644 --- a/lib/LiveUpdate/serialize_tcp.cpp +++ b/lib/LiveUpdate/serialize_tcp.cpp @@ -303,4 +303,8 @@ namespace liu { return deserialize_connection(ent->vla, tcp); } + net::Stream_ptr Restore::as_tcp_stream (net::TCP& tcp) const + { + return std::make_unique (as_tcp_connection(tcp)); + } } diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 4f3762df9d..0a0e6ea6f8 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -29,7 +29,7 @@ using namespace net::tcp; using namespace std; int Connection::Stream::get_cpuid() const noexcept { - return tcp->host().get_cpuid(); + return m_tcp->host().get_cpuid(); } Connection::Connection(TCP& host, Socket local, Socket remote, ConnectCallback callback) From 578cf64dd2d701dafdbb31a1c7e6e3f242857f07 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 27 Apr 2018 22:39:05 +0200 Subject: [PATCH 226/723] musl: Prerequisites for Lua 5 --- src/crt/c_abi.c | 33 ++++++++++++++++++++++++++++++++- src/musl/CMakeLists.txt | 10 ++++++++++ src/musl/dup3.cpp | 31 +++++++++++++++++++++++++++++++ src/musl/execve.cpp | 15 +++++++++++++++ src/musl/getgid.cpp | 13 +++++++++++++ src/musl/kill.cpp | 6 +++--- src/musl/rename.cpp | 13 +++++++++++++ src/musl/select.cpp | 10 +++++----- src/musl/setgid.cpp | 14 ++++++++++++++ src/musl/setpgid.cpp | 14 ++++++++++++++ src/musl/setrlimit.cpp | 2 +- src/musl/setsid.cpp | 14 ++++++++++++++ src/musl/setuid.cpp | 14 ++++++++++++++ src/musl/wait4.cpp | 17 +++++++++++++++++ 14 files changed, 196 insertions(+), 10 deletions(-) create mode 100644 src/musl/dup3.cpp create mode 100644 src/musl/execve.cpp create mode 100644 src/musl/getgid.cpp create mode 100644 src/musl/rename.cpp create mode 100644 src/musl/setgid.cpp create mode 100644 src/musl/setpgid.cpp create mode 100644 src/musl/setsid.cpp create mode 100644 src/musl/setuid.cpp create mode 100644 src/musl/wait4.cpp diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index 21a59c2770..f19974dc8a 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -19,8 +19,10 @@ #include #include #include +#define weak_alias(name, aliasname) _weak_alias (name, aliasname) +#define _weak_alias(name, aliasname) \ + extern __typeof (name) aliasname __attribute__ ((weak, alias (#name))); #define HEAP_ALIGNMENT 63 - void* __dso_handle; uint32_t _move_symbols(void* sym_loc) @@ -88,6 +90,13 @@ int __vsprintf_chk(char* s, int flag, size_t slen, const char* format, va_list a assert ((size_t) res < slen); return res; } +int __vsnprintf_chk (char *s, size_t maxlen, int flags, size_t slen, + const char *format, va_list args) +{ + assert (slen < maxlen); + (void) flags; + return vsnprintf(s, slen, format, args); +} __attribute__((format(printf, 4, 5))) int __sprintf_chk(char* s, int flags, size_t slen, const char *format, ...) { @@ -98,6 +107,18 @@ int __sprintf_chk(char* s, int flags, size_t slen, const char *format, ...) va_end (arg); return done; } +int __snprintf_chk (char *s, size_t maxlen, int flags, size_t slen, + const char *format, ...) +{ + va_list arg; + int done; + + va_start (arg, format); + done = __vsnprintf_chk (s, maxlen, flags, slen, format, arg); + va_end (arg); + + return done; +} int __isoc99_scanf (const char *format, ...) { @@ -117,3 +138,13 @@ int __isoc99_sscanf (const char *s, const char *format, ...) va_end (arg); return done; } + +// TODO: too complicated to implement +#include + +__attribute__ ((noreturn, weak)) +void __longjmp_chk(jmp_buf env, int val) +{ + longjmp(env, val); + __builtin_unreachable(); +} diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index 66a94e647f..7f4ff26637 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -14,6 +14,8 @@ set(MUSL_OBJECTS chmod.cpp chown.cpp cwd.cpp + dup3.cpp # also dup, dup2 + execve.cpp fchmod.cpp fchmodat.cpp fchown.cpp @@ -21,15 +23,23 @@ set(MUSL_OBJECTS ftruncate.cpp getdents.cpp geteuid.cpp + getgid.cpp + getuid.cpp kill.cpp mkdir.cpp mkdirat.cpp mknodat.cpp openat.cpp readlink.cpp + rename.cpp rmdir.cpp select.cpp + setgid.cpp + setpgid.cpp setrlimit.cpp + setsid.cpp + setuid.cpp + wait4.cpp umask.cpp uname.cpp utimensat.cpp diff --git a/src/musl/dup3.cpp b/src/musl/dup3.cpp new file mode 100644 index 0000000000..cfcb1ac81f --- /dev/null +++ b/src/musl/dup3.cpp @@ -0,0 +1,31 @@ +#include "stub.hpp" +#include +#include + +static long sys_dup(int /*oldfd*/) +{ + return -ENOSYS; +} +static long sys_dup2(int /*oldfd*/, int /*newfd*/) +{ + return -ENOSYS; +} +static long sys_dup3(int /*oldfd*/, int /*newfd*/, int /*flags*/) +{ + return -ENOSYS; +} + +extern "C" +long syscall_SYS_dup(int oldfd) { + return stubtrace(sys_dup, "dup", oldfd); +} + +extern "C" +long syscall_SYS_dup2(int oldfd, int newfd) { + return stubtrace(sys_dup2, "dup2", oldfd, newfd); +} + +extern "C" +long syscall_SYS_dup3(int oldfd, int newfd, int flags) { + return stubtrace(sys_dup3, "dup3", oldfd, newfd, flags); +} diff --git a/src/musl/execve.cpp b/src/musl/execve.cpp new file mode 100644 index 0000000000..6d2e231b0c --- /dev/null +++ b/src/musl/execve.cpp @@ -0,0 +1,15 @@ +#include "stub.hpp" +#include +#include + +static long sys_execve(const char*, char *const[], char *const[]) +{ + return -ENOSYS; +} + +extern "C" +long syscall_SYS_execve(const char *filename, char *const argv[], + char *const envp[]) +{ + return stubtrace(sys_execve, "execve", filename, argv, envp); +} diff --git a/src/musl/getgid.cpp b/src/musl/getgid.cpp new file mode 100644 index 0000000000..c7b03a3bbd --- /dev/null +++ b/src/musl/getgid.cpp @@ -0,0 +1,13 @@ +#include "common.hpp" +#include + +static long sys_getgid() +{ + return 0; +} + +extern "C" +long syscall_SYS_getgid() +{ + return strace(sys_getgid, "getgid"); +} diff --git a/src/musl/kill.cpp b/src/musl/kill.cpp index 5abf4e3d1f..5c4e10bd1e 100644 --- a/src/musl/kill.cpp +++ b/src/musl/kill.cpp @@ -1,15 +1,15 @@ #include "common.hpp" #include -int sys_kill(pid_t pid, int sig) { +int sys_kill(pid_t /*pid*/, int /*sig*/) { panic("KILL called"); } -int sys_tkill(int tid, int sig) { +int sys_tkill(int /*tid*/, int /*sig*/) { panic("TKILL called"); } -int sys_tgkill(int tgid, int tid, int sig) { +int sys_tgkill(int /*tgid*/, int /*tid*/, int /*sig*/) { panic("TGKILL called"); } diff --git a/src/musl/rename.cpp b/src/musl/rename.cpp new file mode 100644 index 0000000000..df26423e48 --- /dev/null +++ b/src/musl/rename.cpp @@ -0,0 +1,13 @@ +#include "stub.hpp" +#include + +static long sys_rename(const char* /*oldpath*/, const char* /*newpath*/) +{ + return -ENOSYS; +} + +extern "C" +long syscall_SYS_rename(const char* oldpath, const char* newpath) +{ + return stubtrace(sys_rename, "rename", oldpath, newpath); +} diff --git a/src/musl/select.cpp b/src/musl/select.cpp index c2dfbb05e2..af9ee8e9cb 100644 --- a/src/musl/select.cpp +++ b/src/musl/select.cpp @@ -2,11 +2,11 @@ #include #include -long sys_select(int nfds, - fd_set* readfds, - fd_set* writefds, - fd_set* exceptfds, - struct timeval* timeout) +long sys_select(int /*nfds*/, + fd_set* /*readfds*/, + fd_set* /*writefds*/, + fd_set* /*exceptfds*/, + struct timeval* /*timeout*/) { return -ENOSYS; } diff --git a/src/musl/setgid.cpp b/src/musl/setgid.cpp new file mode 100644 index 0000000000..b57bf32c19 --- /dev/null +++ b/src/musl/setgid.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include +#include + +long sys_setgid(gid_t /*gid*/) +{ + return -ENOSYS; +} + +extern "C" +long syscall_SYS_setgid(gid_t gid) +{ + return strace(sys_setgid, "setgid", gid); +} diff --git a/src/musl/setpgid.cpp b/src/musl/setpgid.cpp new file mode 100644 index 0000000000..8055a09a3c --- /dev/null +++ b/src/musl/setpgid.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include +#include + +long sys_setpgid(pid_t /*pid*/, gid_t /*gid*/) +{ + return -ENOSYS; +} + +extern "C" +long syscall_SYS_setpgid(pid_t pid, gid_t gid) +{ + return strace(sys_setpgid, "setpgid", pid, gid); +} diff --git a/src/musl/setrlimit.cpp b/src/musl/setrlimit.cpp index e51bdd192b..f7bc24ee38 100644 --- a/src/musl/setrlimit.cpp +++ b/src/musl/setrlimit.cpp @@ -1,7 +1,7 @@ #include "common.hpp" #include -long sys_setrlimit(int resource, const struct rlimit *rlim) +long sys_setrlimit(int /*resource*/, const struct rlimit* /*rlim*/) { return -ENOSYS; } diff --git a/src/musl/setsid.cpp b/src/musl/setsid.cpp new file mode 100644 index 0000000000..75a984de48 --- /dev/null +++ b/src/musl/setsid.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include +#include + +long sys_setsid() +{ + return 0; +} + +extern "C" +long syscall_SYS_setsid() +{ + return strace(sys_setsid, "setsid"); +} diff --git a/src/musl/setuid.cpp b/src/musl/setuid.cpp new file mode 100644 index 0000000000..9664829c9a --- /dev/null +++ b/src/musl/setuid.cpp @@ -0,0 +1,14 @@ +#include "common.hpp" +#include +#include + +long sys_setuid(uid_t /*uid*/) +{ + return -ENOSYS; +} + +extern "C" +long syscall_SYS_setuid(uid_t uid) +{ + return strace(sys_setuid, "setuid", uid); +} diff --git a/src/musl/wait4.cpp b/src/musl/wait4.cpp new file mode 100644 index 0000000000..d6edcfa483 --- /dev/null +++ b/src/musl/wait4.cpp @@ -0,0 +1,17 @@ +#include "common.hpp" +#include +#include + +static long +sys_wait4(pid_t /*pid*/, int* /*wstatus*/, int /*options*/, + struct rusage* /*rusage*/) +{ + return 0; +} + +extern "C" +long syscall_SYS_wait4(pid_t pid, int *wstatus, int options, + struct rusage *rusage) +{ + return strace(sys_wait4, "wait4", pid, wstatus, options, rusage); +} From e013d0261a8da41e29fb2b7d56f392fa245defa8 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 27 Apr 2018 22:39:35 +0200 Subject: [PATCH 227/723] examples: Add lua5.3 example service --- examples/lua5/CMakeLists.txt | 33 ++++++++++++++++++++++ examples/lua5/README.md | 3 ++ examples/lua5/service.cpp | 54 ++++++++++++++++++++++++++++++++++++ examples/lua5/setup.sh | 2 ++ 4 files changed, 92 insertions(+) create mode 100644 examples/lua5/CMakeLists.txt create mode 100644 examples/lua5/README.md create mode 100644 examples/lua5/service.cpp create mode 100755 examples/lua5/setup.sh diff --git a/examples/lua5/CMakeLists.txt b/examples/lua5/CMakeLists.txt new file mode 100644 index 0000000000..05c3e5bb65 --- /dev/null +++ b/examples/lua5/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 2.8.9) + +# IncludeOS install location +if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) + set(ENV{INCLUDEOS_PREFIX} /usr/local) +endif() +include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) +project (tcp) + +# Human-readable name of your service +set(SERVICE_NAME "Lua5 Example Service") + +# Name of your service binary +set(BINARY "lua5") + +# Source files to be linked with OS library parts to form bootable image +set(SOURCES + service.cpp # ...add more here + ) + +set(LOCAL_INCLUDES /usr/include/x86_64-linux-gnu /usr/include/lua5.3) + +set(DRIVERS + #virtionet + ) + +set(PLUGINS + ) + +set(LIBRARIES /usr/lib/x86_64-linux-gnu/liblua5.3.a) + +# include service build script +include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/examples/lua5/README.md b/examples/lua5/README.md new file mode 100644 index 0000000000..2b7037bd95 --- /dev/null +++ b/examples/lua5/README.md @@ -0,0 +1,3 @@ +### Lua 5.3 + +A service with Lua 5.3 statically linked diff --git a/examples/lua5/service.cpp b/examples/lua5/service.cpp new file mode 100644 index 0000000000..40941e5a0f --- /dev/null +++ b/examples/lua5/service.cpp @@ -0,0 +1,54 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +extern "C" { + #include "lua.h" + #include "lualib.h" + #include "lauxlib.h" +} + +const std::string lua_script = R"N0TLU4( + + mytable = setmetatable({key1 = "value1"}, { + __index = function(mytable, key) + + if key == "key2" then + return "metatablevalue" + else + return mytable[key] + end + end + }) + + print(mytable.key1,mytable.key2) + +)N0TLU4"; + +void Service::start() +{ + // initialization + lua_State * L = luaL_newstate(); + luaL_openlibs(L); + + // execute script + int load_stat = luaL_loadbuffer(L,lua_script.c_str(), lua_script.size(), "test"); + lua_pcall(L, 0, 0, 0); + + // cleanup + lua_close(L); +} diff --git a/examples/lua5/setup.sh b/examples/lua5/setup.sh new file mode 100755 index 0000000000..8ea139e87c --- /dev/null +++ b/examples/lua5/setup.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sudo apt install -y liblua5.3-dev From 76d458748be525bf8ed771901ec59039f48b03d9 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 30 Apr 2018 11:23:22 +0200 Subject: [PATCH 228/723] openssl: Implement weak serialize on TLS_stream --- src/net/openssl/server.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/net/openssl/server.cpp b/src/net/openssl/server.cpp index 7403f7405c..6d4dfcb177 100644 --- a/src/net/openssl/server.cpp +++ b/src/net/openssl/server.cpp @@ -5,6 +5,11 @@ namespace openssl { + __attribute__((weak)) + size_t TLS_stream::serialize_to(void*) const { + return 0; + } + // https://gist.github.com/darrenjs/4645f115d10aa4b5cebf57483ec82eca inline static void handle_error(const char* file, int lineno, const char* msg) { From 4b868925323dcaf3787cb309b4ef5a2358f074ce Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 30 Apr 2018 11:51:07 +0200 Subject: [PATCH 229/723] test: Run prereq.sh if one exists in each example folder --- examples/lua5/{setup.sh => prereq.sh} | 0 test/misc/build_examples/test.sh | 4 ++++ 2 files changed, 4 insertions(+) rename examples/lua5/{setup.sh => prereq.sh} (100%) diff --git a/examples/lua5/setup.sh b/examples/lua5/prereq.sh similarity index 100% rename from examples/lua5/setup.sh rename to examples/lua5/prereq.sh diff --git a/test/misc/build_examples/test.sh b/test/misc/build_examples/test.sh index 3baad10653..ef4b01e73e 100755 --- a/test/misc/build_examples/test.sh +++ b/test/misc/build_examples/test.sh @@ -46,6 +46,10 @@ function build_service() { str=">>> Now making $BASE" printf "%-50s " "* $BASE" git submodule update --init --recursive + if [ -e prereq.sh ] + then + ./prereq.sh + fi $INCLUDEOS_PREFIX/bin/boot -cb . &> $tmpfile echo "[ PASS ]" } From 5e61dde176d62577d997abf974961e3ab26636c5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 3 May 2018 09:09:52 +0200 Subject: [PATCH 230/723] net: Work on websocket, streams --- api/fs/common.hpp | 5 +++ api/net/botan/credman.hpp | 6 ++-- api/net/botan/tls_server.hpp | 9 +++-- api/net/ws/websocket.hpp | 5 +-- src/net/https/botan_server.cpp | 7 ++-- src/net/openssl/server.cpp | 5 +++ src/net/ws/websocket.cpp | 60 ++++++++++++++-------------------- 7 files changed, 49 insertions(+), 48 deletions(-) diff --git a/api/fs/common.hpp b/api/fs/common.hpp index e9802d356a..e2ecaa25fc 100644 --- a/api/fs/common.hpp +++ b/api/fs/common.hpp @@ -151,6 +151,11 @@ namespace fs { error_t error() const noexcept { return err_; } + /** + * Returns the underlying buffer + **/ + auto& get() noexcept { return this->buffer_; } + /** * @brief Get the starting address of the underlying data buffer * diff --git a/api/net/botan/credman.hpp b/api/net/botan/credman.hpp index b60eb5d8d0..7ffb229872 100644 --- a/api/net/botan/credman.hpp +++ b/api/net/botan/credman.hpp @@ -103,7 +103,7 @@ class Credman : public Botan::Credentials_Manager * 4. create certificate request with private key 2 * 5. create CA with key and cert * 6, create certificate by signing - * + * **/ inline Credman* Credman::create( const std::string& server_name, @@ -112,7 +112,7 @@ inline Credman* Credman::create( Botan::X509_Certificate ca_cert, std::unique_ptr server_key) { - Botan::X509_CA ca(ca_cert, *ca_key, "SHA-256", rng); + Botan::X509_CA ca(ca_cert, *ca_key, "SHA-512", rng); // create server certificate from CA auto now = std::chrono::system_clock::now(); @@ -124,7 +124,7 @@ inline Credman* Credman::create( server_opts.common_name = server_name; server_opts.country = "VT"; - auto req = Botan::X509::create_cert_req(server_opts, *server_key, "SHA-256", rng); + auto req = Botan::X509::create_cert_req(server_opts, *server_key, "SHA-512", rng); auto server_cert = ca.sign_request(req, rng, start_time, end_time); diff --git a/api/net/botan/tls_server.hpp b/api/net/botan/tls_server.hpp index 8d14cc9218..615b6e0ded 100644 --- a/api/net/botan/tls_server.hpp +++ b/api/net/botan/tls_server.hpp @@ -76,9 +76,13 @@ class Server : public Botan::TLS::Callbacks, public net::Stream void close() override { m_transport->close(); + CloseCallback cb = std::move(m_on_close); + reset_callbacks(); + if (cb) cb(); } void abort() override { m_transport->abort(); + this->close(); } void reset_callbacks() override { @@ -142,11 +146,6 @@ class Server : public Botan::TLS::Callbacks, public net::Stream printf("Fatal TLS error %s\n", e.what()); this->close(); } - catch(...) - { - printf("Unknown error!\n"); - this->close(); - } } void tls_alert(Botan::TLS::Alert alert) override diff --git a/api/net/ws/websocket.hpp b/api/net/ws/websocket.hpp index 47739b607f..8889da13c2 100644 --- a/api/net/ws/websocket.hpp +++ b/api/net/ws/websocket.hpp @@ -225,6 +225,7 @@ class WebSocket { // close the websocket void close(); + void close(uint16_t reason); // user callbacks close_func on_close = nullptr; @@ -268,7 +269,6 @@ class WebSocket { static std::pair deserialize_from(const void*); WebSocket(net::Stream_ptr, bool); - WebSocket(WebSocket&&); ~WebSocket(); private: @@ -279,12 +279,13 @@ class WebSocket { bool clientside; WebSocket(const WebSocket&) = delete; + WebSocket(WebSocket&&) = delete; WebSocket& operator= (const WebSocket&) = delete; WebSocket& operator= (WebSocket&&) = delete; void read_data(net::tcp::buffer_t); bool write_opcode(op_code code, const char*, size_t); void failure(const std::string&); - void tcp_closed(); + void close_callback_once(uint16_t code); size_t create_message(const uint8_t*, size_t len); void finalize_message(); void reset(); diff --git a/src/net/https/botan_server.cpp b/src/net/https/botan_server.cpp index 6c31a997a3..217666da2c 100644 --- a/src/net/https/botan_server.cpp +++ b/src/net/https/botan_server.cpp @@ -46,8 +46,9 @@ namespace http { // load CA certificate assert(file_ca_cert.is_valid()); - auto ca_cert = file_ca_cert.read(); - std::vector vca_cert(ca_cert.begin(), ca_cert.end()); + auto ca_cert = file_ca_cert.read(0, file_ca_cert.size()); + assert(ca_cert.is_valid()); + // load CA private key auto ca_key = read_pkey(file_ca_key); // load server private key @@ -57,7 +58,7 @@ namespace http server_name, get_rng(), std::move(ca_key), - Botan::X509_Certificate(vca_cert), + Botan::X509_Certificate(*ca_cert.get()), std::move(srv_key)); this->credman.reset(credman); diff --git a/src/net/openssl/server.cpp b/src/net/openssl/server.cpp index 7403f7405c..f4105eac30 100644 --- a/src/net/openssl/server.cpp +++ b/src/net/openssl/server.cpp @@ -77,4 +77,9 @@ namespace openssl assert(error == SSL_ERROR_NONE); return ctx; } + + __attribute__((weak)) + size_t TLS_stream::serialize_to(void*) const { + return 0; + } } diff --git a/src/net/ws/websocket.cpp b/src/net/ws/websocket.cpp index 6892a325d5..062c6aa710 100644 --- a/src/net/ws/websocket.cpp +++ b/src/net/ws/websocket.cpp @@ -304,15 +304,13 @@ void WebSocket::finalize_message() if (hdr.data_length() >= 2) { // provide reason to user uint16_t reason = *(uint16_t*) message->data(); - if (this->on_close) - this->on_close(__builtin_bswap16(reason)); + this->close(reason); } else { - if (this->on_close) this->on_close(1000); + this->close(1000); } - // close it down - this->close(); - break; + // the websocket is DEAD after close() + return; case op_code::PING: if (on_ping(hdr.data(), hdr.data_length())) // if return true, pong back write_opcode(op_code::PONG, hdr.data(), hdr.data_length()); @@ -419,49 +417,39 @@ bool WebSocket::write_opcode(op_code code, const char* buffer, size_t datalen) this->stream->write(buffer, datalen); return true; } -void WebSocket::tcp_closed() -{ - if (this->on_close != nullptr) this->on_close(1000); - this->reset(); -} WebSocket::WebSocket(net::Stream_ptr stream_ptr, bool client) : stream(std::move(stream_ptr)), max_msg_size(0), clientside(client) { assert(stream != nullptr); this->stream->on_read(8*1024, {this, &WebSocket::read_data}); - this->stream->on_close({this, &WebSocket::tcp_closed}); -} - -WebSocket::WebSocket(WebSocket&& other) -{ - on_close = std::move(other.on_close); - on_error = std::move(other.on_error); - on_read = std::move(other.on_read); - on_ping = std::move(other.on_ping); - on_pong = std::move(other.on_pong); - on_pong_timeout = std::move(other.on_pong_timeout); - - stream = std::move(other.stream); - clientside = other.clientside; - other.ping_timer.stop(); // .. - - max_msg_size = other.max_msg_size; + this->stream->on_close({this, &WebSocket::close}); } WebSocket::~WebSocket() { if (stream != nullptr && stream->is_connected()) - this->close(); + this->close(1000); } void WebSocket::close() { + this->close(1000); +} +void WebSocket::close(const uint16_t reason) +{ + assert(stream != nullptr); /// send CLOSE message if (this->stream->is_writable()) - this->write_opcode(op_code::CLOSE, nullptr, 0); + this->write_opcode(op_code::CLOSE, "Closed", 6); /// close and unset socket this->stream->close(); + this->close_callback_once(reason); +} +void WebSocket::close_callback_once(const uint16_t reason) +{ + auto close_func = std::move(this->on_close); this->reset(); + if (close_func) close_func(reason); } void WebSocket::reset() @@ -472,15 +460,17 @@ void WebSocket::reset() this->on_ping = nullptr; this->on_pong = nullptr; this->on_pong_timeout = nullptr; - ping_timer.stop(); - stream->reset_callbacks(); - stream->close(); - stream = nullptr; + this->ping_timer.stop(); + if (this->stream != nullptr) + { + this->stream->reset_callbacks(); + this->stream = nullptr; + } } void WebSocket::failure(const std::string& reason) { - if (stream != nullptr) stream->close(); + if (this->stream != nullptr) this->stream->close(); if (this->on_error) on_error(reason); } From fd8ba354cc4a099b6c190859ce037dba742f7590 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 3 May 2018 09:41:07 +0200 Subject: [PATCH 231/723] tcp: Work on SACK, works almost always --- api/net/tcp/connection.hpp | 6 ++- api/net/tcp/sack.hpp | 16 +++++++- examples/TCP_perf/service.cpp | 19 +++++++--- src/net/tcp/connection.cpp | 69 +++++++++++++++++++---------------- src/net/tcp/read_buffer.cpp | 4 +- src/net/tcp/read_request.cpp | 5 +-- 6 files changed, 74 insertions(+), 45 deletions(-) diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index 9b9aea6827..97c9f92154 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -608,6 +608,9 @@ class Connection { Socket remote() const noexcept { return remote_; } + auto bytes_sacked() const noexcept + { return bytes_sacked_; } + /** * @brief Interface for one of the many states a Connection can have. @@ -837,10 +840,11 @@ class Connection { /** State if connection is in TCP write queue or not. */ bool queued_; - using Sack_list = sack::List>; + using Sack_list = sack::List>; std::unique_ptr sack_list; /** If SACK is permitted (option has been seen from peer) */ bool sack_perm = false; + size_t bytes_sacked_ = 0; /** Congestion control */ // is fast recovery state diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index f698350c17..a416b2e3a1 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -117,6 +117,9 @@ class List { Entries recent_entries() const noexcept { return impl.recent_entries(); } + bool contains(const seq_t seq) const noexcept + { return impl.contains(seq); } + List_impl impl; }; @@ -200,7 +203,7 @@ class Fixed_list { Ack_result new_valid_ack(const seq_t seq, size_t len) { - const auto ack = seq + len; + const seq_t ack = seq + (uint32_t)len; uint32_t bytes_freed = 0; for(auto it = blocks.begin(); it != blocks.end(); it++) @@ -208,7 +211,7 @@ class Fixed_list { if (it->contains(ack)) { bytes_freed = it->size(); - len -= (ack - it->start); + len -= (ack - it->start); // result in 0 if not partial blocks.erase(it); break; } @@ -235,6 +238,15 @@ class Fixed_list { blocks.splice(blocks.begin(), blocks, it); } + bool contains(const seq_t seq) const noexcept + { + for(auto& block : blocks) { + if(block.contains(seq)) + return true; + } + return false; + } + List blocks; }; diff --git a/examples/TCP_perf/service.cpp b/examples/TCP_perf/service.cpp index 646db96075..874be7876b 100644 --- a/examples/TCP_perf/service.cpp +++ b/examples/TCP_perf/service.cpp @@ -24,6 +24,7 @@ using namespace net::tcp; +size_t bufsize = 128*1024; uint32_t SIZE = 1024*1024*512; uint64_t packets_rx{0}; uint64_t packets_tx{0}; @@ -33,6 +34,7 @@ uint8_t wscale{5}; bool timestamps{true}; std::chrono::milliseconds dack{40}; uint64_t ts = 0; +bool SACK{true}; struct activity { void reset() { @@ -63,8 +65,11 @@ void start_measure() received = 0; packets_rx = Statman::get().get_by_name("eth0.ethernet.packets_rx").get_uint64(); packets_tx = Statman::get().get_by_name("eth0.ethernet.packets_tx").get_uint64(); - printf(" DACK: %lli ms WSIZE: %u WS: %u CALC_WIN: %u TS: %s\n", - dack.count(), winsize, wscale, winsize << wscale, timestamps ? "ON" : "OFF"); + printf(" BUFSZ=%zukB DACK=%llims WSIZE=%u WS=%u CALC_WIN=%ukB TS=%s SACK=%s\n", + bufsize/1024, + dack.count(), winsize, wscale, (winsize << wscale)/1024, + timestamps ? "ON" : "OFF", + SACK ? "ON" : "OFF"); ts = OS::nanos_since_boot(); activity_before.reset(); } @@ -79,10 +84,10 @@ void stop_measure() packets_rx = Statman::get().get_by_name("eth0.ethernet.packets_rx").get_uint64() - packets_rx; packets_tx = Statman::get().get_by_name("eth0.ethernet.packets_tx").get_uint64() - packets_tx; - printf("Packets RX [%llu] TX [%llu]\n", packets_rx, packets_tx); + printf("Packets RX [%lu] TX [%lu]\n", packets_rx, packets_tx); double durs = (double) diff / 1000000000ULL; double mbits = (received/(1024*1024)*8) / durs; - printf("Duration: %.2fs - Payload: %lld/%u MB - %.2f MBit/s\n", + printf("Duration: %.2fs - Payload: %lu/%u MB - %.2f MBit/s\n", durs, received/(1024*1024), SIZE/(1024*1024), mbits); } @@ -111,6 +116,7 @@ void Service::ready() tcp.set_window_size(winsize, wscale); tcp.set_timestamps(timestamps); + tcp.set_SACK(SACK); tcp.listen(1337).on_connect([](Connection_ptr conn) { @@ -146,12 +152,15 @@ void Service::ready() conn->on_disconnect([] (auto self, auto reason) { (void) reason; + if(const auto bytes_sacked = self->bytes_sacked(); bytes_sacked) + printf("SACK: %zu bytes (%zu kB)\n", bytes_sacked, bytes_sacked/(1024)); + if(!self->is_closing()) self->close(); stop_measure(); }); - conn->on_read(16384, [] (buffer_t buf) + conn->on_read(bufsize, [] (buffer_t buf) { recv(buf->size()); }); diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 5e96750085..9da1119e1e 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -25,13 +25,13 @@ #include #include -/*#include // remove me, sack debugging +#include // remove me, sack debugging std::ostream& operator<< (std::ostream& out, const net::tcp::sack::Entries& ent) { for (auto el : ent) { out << el << "\n"; } return out; -}*/ +} using namespace net::tcp; @@ -701,34 +701,22 @@ void Connection::recv_data(const Packet& in) { size_t length = in.tcp_data_length(); - //auto& front = read_request->front(); - //if(front.missing()) { - // printf(" In order: SEQ=%u hole=%u fits=%u sz=%u front_start=%u front_end=%u\n", - // in.seq(), front.missing(), read_request->fits(in.seq()), read_request->size(), front.start_seq(), front.end_seq()); - //} - // If we had packet loss before (and SACK is on) // we need to clear up among the blocks // and increase the total amount of bytes acked if(UNLIKELY(sack_list)) { - //printf("In order: SEQ=%u sz=%lu - front: hole=%u fits=%u sz=%u cap=%u start=%u end=%u\n", - // in.seq(), length, front.missing(), read_request->fits(in.seq()), - // front.size(), front.capacity(), front.start_seq(), front.end_seq()); - - auto res = sack_list->new_valid_ack(in.seq(), length); + const auto res = sack_list->new_valid_ack(in.seq(), length); // if any bytes are cleared up in sack, increase expected sequence number cb.RCV.NXT += res.blocksize; - if(UNLIKELY(length != res.length)) - printf("mismatch len=%u res.len=%u\n", length, res.length); // TODO: This Ensures verifies that we never have partial packets - Ensures(length == res.length); - //length = res.length; - - if(cb.RCV.NXT != in.seq()) - printf("ACKED res.bytes=%u\n", res.blocksize); - //std::cout << sack_list->recent_entries(); + if(length != res.length) { + printf("len=%zu res.len=%zu blksz=%u\n seq=%u", + length, res.length, res.blocksize, in.seq()); + std::cout << sack_list->recent_entries() << "\n"; + } + assert(length == res.length && "No partial insertion support"); } const auto recv = read_request->insert(in.seq(), in.tcp_data(), length, in.isset(PSH)); @@ -749,6 +737,15 @@ void Connection::recv_data(const Packet& in) // [RFC 5681] ??? } +// This function need to sync both SACK and the read buffer, meaning: +// * Data cannot be old segments (already acked) +// * Data cannot be duplicate (already S-acked) +// * Read buffer needs to have room for the data +// * SACK list needs to have room for the entry (either connects or new) +// +// For now, only full segments are allowed (not partial), +// meaning the data will get thrown away if the read buffer not fully fits it. +// This makes everything much easier. void Connection::recv_out_of_order(const Packet& in) { // Packets before this point would totally ruin the buffer @@ -763,25 +760,33 @@ void Connection::recv_out_of_order(const Packet& in) const size_t length = in.tcp_data_length(); auto seq = in.seq(); - auto fits = read_request->fits(seq); - //if(fits) - // printf("Out-of-order: SEQ=%u REL=%u RCV.NXT=%u fits=%u\n", - // in.seq(), in.seq() - cb.RCV.NXT, cb.RCV.NXT, fits); + // If it's already SACKed, it means we already received the data, + // just ignore + // note: Due to not accepting partial data i think we're safe with + // not passing length as a second arg to contains + if(UNLIKELY(sack_list->contains(seq))) { + return; + } + + auto fits = read_request->fits(seq); // TODO: if our packet partial fits, we just ignores it for now // to avoid headache if(fits >= length) { - auto inserted = read_request->insert(seq, in.tcp_data(), length, in.isset(PSH)); - Ensures(inserted == length); - - // Assume for now that we have room in sack list - auto res = sack_list->recv_out_of_order(seq, inserted); + // Insert into SACK list before the buffer, since we already know 'fits' + auto res = sack_list->recv_out_of_order(seq, length); - Ensures(res.length == length); + // if we can't add the entry, we can't allow to insert into the buffer + if(res.length != length) { + Ensures(res.length == 0 && "No partial insertion support"); + return; + } - //std::cout << sack_list->recent_entries(); + const auto inserted = read_request->insert(seq, in.tcp_data(), length, in.isset(PSH)); + Ensures(inserted == length && "No partial insertion support"); + bytes_sacked_ += inserted; } /* diff --git a/src/net/tcp/read_buffer.cpp b/src/net/tcp/read_buffer.cpp index dc1c84a71e..807de66fe3 100644 --- a/src/net/tcp/read_buffer.cpp +++ b/src/net/tcp/read_buffer.cpp @@ -32,7 +32,7 @@ size_t Read_buffer::insert(const seq_t seq, const uint8_t* data, size_t len, boo assert(buf != nullptr && "Buffer seems to be stolen, make sure to renew()"); // get the relative sequence number (the diff) - size_t rel = seq - start; + size_t rel = (seq_t)(seq - start); assert(rel < capacity() && "No point trying to write at or above the end"); //printf("seq=%u, start=%u, rel: %lu sz=%lu\n", seq, start, rel, size()); @@ -40,7 +40,7 @@ size_t Read_buffer::insert(const seq_t seq, const uint8_t* data, size_t len, boo len = std::min(capacity() - rel, len); // fill/add hole - hole += (rel >= buf->size()) ? rel - buf->size() : -len; + hole += (rel >= buf->size()) ? (rel - buf->size()) : -len; assert(hole >= 0 && "A hole cannot have a negative depth.."); // add data to the buffer at the relative position diff --git a/src/net/tcp/read_request.cpp b/src/net/tcp/read_request.cpp index 1dd02715e3..bdaf8d8aa8 100644 --- a/src/net/tcp/read_request.cpp +++ b/src/net/tcp/read_request.cpp @@ -77,13 +77,12 @@ namespace tcp { // maybe it isnt even necessary..) else { + //printf("size=%zu rem=%zu\n", buffers.size(), rem); // if there are other buffers following this one, // and the buffer wasnt full, fill the small gap if(UNLIKELY(rem != 0)) { - buf->reset(seq, rem); - //printf("remaining=%u, reset start=%u end=%u\n", - // rem, buf->start_seq(), buf->end_seq()); + buf->reset(buf->end_seq()-(uint32_t)rem, rem); Ensures(buf->end_seq() == buffers.at(1)->start_seq()); } else From 57123510b0bedb0696a1e6c9674b93c51fa8ea2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 4 May 2018 12:58:31 +0200 Subject: [PATCH 232/723] tcp: Support recv partial SACKed payload --- api/net/tcp/sack.hpp | 60 ++++++++++++++++++++++++++++++++++---- src/net/tcp/connection.cpp | 26 ++++++++--------- 2 files changed, 66 insertions(+), 20 deletions(-) diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index a416b2e3a1..d851b1d483 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -81,11 +81,14 @@ struct Block { Block& operator=(uint64_t whole) { this->whole = whole; return *this; } + std::string to_string() const + { return "[" + std::to_string(start) + " => " + std::to_string(end) + "]"; } + }__attribute__((packed)); // Print for Block inline std::ostream& operator<<(std::ostream& out, const Block& b) { - out << "[" << b.start << " => " << b.end << "]"; + out << b.to_string(); return out; } @@ -120,6 +123,9 @@ class List { bool contains(const seq_t seq) const noexcept { return impl.contains(seq); } + bool contains(const seq_t seq, const uint32_t len) const noexcept + { return impl.contains(seq, len); } + List_impl impl; }; @@ -152,6 +158,37 @@ connects_to(Iterator first, Iterator last, const Connectable& value) return connected; } +template +struct Partial_connect_result { + Iterator end; + Iterator start; + uint32_t offs_start; +}; + +template +Partial_connect_result +partial_connects_to(Iterator first, Iterator last, const Connectable& value) +{ + Partial_connect_result connected{last, last, 0}; + for (; first != last; ++first) + { + if (first->connects_end(value)) + { + connected.end = first; + } + else if (first->contains(value.end)) + { + connected.start = first; + connected.offs_start = value.end - first->start; + } + + // if we connected to two nodes, no point in looking for more + if (connected.start != last and connected.end != last) + break; + } + return connected; +} + template class Fixed_list { public: @@ -163,11 +200,9 @@ class Fixed_list { Ack_result recv_out_of_order(const seq_t seq, size_t len) { - // TODO: This just assumes nothing of the block exists from before. - // Uncertain if this will cause an issue. Block blk{seq, static_cast(seq+len)}; - auto connected = connects_to(blocks.begin(), blocks.end(), blk); + auto connected = partial_connects_to(blocks.begin(), blocks.end(), blk); if (connected.end != blocks.end()) // Connectes to an end { @@ -177,6 +212,8 @@ class Fixed_list { if (connected.start != blocks.end()) { connected.end->end = connected.start->end; + blk.end -= connected.offs_start; // shrink end with offset (if partial) + Ensures(connected.offs_start <= len && "Offset cannot be bigger than length"); blocks.erase(connected.start); } @@ -185,6 +222,8 @@ class Fixed_list { else if (connected.start != blocks.end()) // Connected only to an start { connected.start->start = blk.start; + blk.end -= connected.offs_start; // shrink end with offset (if partial) + Ensures(connected.offs_start <= len && "Offset cannot be bigger than length"); move_to_front(connected.start); } else // No connection - new entry @@ -197,8 +236,7 @@ class Fixed_list { blocks.push_front(blk); } - // just return the full length - return {len, blk.size()}; + return {blk.size(), 0}; } Ack_result new_valid_ack(const seq_t seq, size_t len) @@ -247,6 +285,16 @@ class Fixed_list { return false; } + bool contains(const seq_t seq, const uint32_t len) const noexcept + { + const auto ack = seq + len; + for(auto& block : blocks) { + if(block.contains(seq) or block.contains(ack)) + return true; + } + return false; + } + List blocks; }; diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 9da1119e1e..6fd20077a5 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -25,13 +25,13 @@ #include #include -#include // remove me, sack debugging +/*#include // remove me, sack debugging std::ostream& operator<< (std::ostream& out, const net::tcp::sack::Entries& ent) { for (auto el : ent) { out << el << "\n"; } return out; -} +}*/ using namespace net::tcp; @@ -710,13 +710,12 @@ void Connection::recv_data(const Packet& in) // if any bytes are cleared up in sack, increase expected sequence number cb.RCV.NXT += res.blocksize; - // TODO: This Ensures verifies that we never have partial packets - if(length != res.length) { - printf("len=%zu res.len=%zu blksz=%u\n seq=%u", - length, res.length, res.blocksize, in.seq()); - std::cout << sack_list->recent_entries() << "\n"; - } - assert(length == res.length && "No partial insertion support"); + // if the ACK was a new ack, but some of the latter part of the segment + // was already received by SACK, it will report a shorter length + // (only the amount not yet received). + // The remaining data is already covered in the reported blocksize + // when incrementing RCV.NXT + length = res.length; } const auto recv = read_request->insert(in.seq(), in.tcp_data(), length, in.isset(PSH)); @@ -758,7 +757,7 @@ void Connection::recv_out_of_order(const Packet& in) if(UNLIKELY(not sack_list)) sack_list = std::make_unique(); - const size_t length = in.tcp_data_length(); + size_t length = in.tcp_data_length(); auto seq = in.seq(); // If it's already SACKed, it means we already received the data, @@ -770,7 +769,6 @@ void Connection::recv_out_of_order(const Packet& in) } auto fits = read_request->fits(seq); - // TODO: if our packet partial fits, we just ignores it for now // to avoid headache if(fits >= length) @@ -779,10 +777,10 @@ void Connection::recv_out_of_order(const Packet& in) auto res = sack_list->recv_out_of_order(seq, length); // if we can't add the entry, we can't allow to insert into the buffer - if(res.length != length) { - Ensures(res.length == 0 && "No partial insertion support"); + length = res.length; + + if(UNLIKELY(length == 0)) return; - } const auto inserted = read_request->insert(seq, in.tcp_data(), length, in.isset(PSH)); Ensures(inserted == length && "No partial insertion support"); From caffc15cbdce0364edaf8514bb4975afaf2c4fb8 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 9 May 2018 12:25:06 +0200 Subject: [PATCH 233/723] net: Don't assume message is both header and data --- api/net/ws/websocket.hpp | 6 +++--- src/net/openssl/server.cpp | 5 ----- src/net/ws/websocket.cpp | 8 +++----- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/api/net/ws/websocket.hpp b/api/net/ws/websocket.hpp index 8889da13c2..03a3852c56 100644 --- a/api/net/ws/websocket.hpp +++ b/api/net/ws/websocket.hpp @@ -206,7 +206,7 @@ class WebSocket { create_response_handler(Connect_handler on_connect, std::string key); void write(const char* buffer, size_t len, op_code = op_code::TEXT); - void write(net::tcp::buffer_t, op_code = op_code::TEXT); + void write(Stream::buffer_t, op_code = op_code::TEXT); void write(const std::string& text) { @@ -221,7 +221,7 @@ class WebSocket { bool ping(Timer::duration_t timeout) { return ping(nullptr, 0, timeout); } - //void ping(net::tcp::buffer_t, Timer::duration_t timeout); + //void ping(Stream::buffer_t, Timer::duration_t timeout); // close the websocket void close(); @@ -282,7 +282,7 @@ class WebSocket { WebSocket(WebSocket&&) = delete; WebSocket& operator= (const WebSocket&) = delete; WebSocket& operator= (WebSocket&&) = delete; - void read_data(net::tcp::buffer_t); + void read_data(Stream::buffer_t); bool write_opcode(op_code code, const char*, size_t); void failure(const std::string&); void close_callback_once(uint16_t code); diff --git a/src/net/openssl/server.cpp b/src/net/openssl/server.cpp index 6b4826c62e..6d4dfcb177 100644 --- a/src/net/openssl/server.cpp +++ b/src/net/openssl/server.cpp @@ -82,9 +82,4 @@ namespace openssl assert(error == SSL_ERROR_NONE); return ctx; } - - __attribute__((weak)) - size_t TLS_stream::serialize_to(void*) const { - return 0; - } } diff --git a/src/net/ws/websocket.cpp b/src/net/ws/websocket.cpp index 062c6aa710..bf287f7e94 100644 --- a/src/net/ws/websocket.cpp +++ b/src/net/ws/websocket.cpp @@ -183,7 +183,7 @@ void WebSocket::connect( WS_client_connector::create_response_handler(std::move(callback), std::move(key))); } -void WebSocket::read_data(net::tcp::buffer_t buf) +void WebSocket::read_data(Stream::buffer_t buf) { // silently ignore data for reset connection if (this->stream == nullptr) return; @@ -334,9 +334,7 @@ static Stream::buffer_t create_wsmsg(size_t len, op_code code, bool client) // generate header length based on buffer length const size_t header_len = net::ws_header::header_length(len, client); // create shared buffer with position at end of header - auto buffer = tcp::construct_buffer(); - buffer->reserve(header_len + len); - buffer->resize(header_len); + auto buffer = tcp::construct_buffer(header_len); // create header on buffer new (buffer->data()) ws_header; auto& hdr = *(ws_header*) buffer->data(); @@ -380,7 +378,7 @@ void WebSocket::write(const char* data, size_t len, op_code code) /// send everything as shared buffer this->stream->write(buf); } -void WebSocket::write(net::tcp::buffer_t buffer, op_code code) +void WebSocket::write(Stream::buffer_t buffer, op_code code) { if (UNLIKELY(this->stream == nullptr)) { failure("write: Already closed"); From 1eef6c2f9d6836e03976626bfe84953337b73f21 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 9 May 2018 12:28:10 +0200 Subject: [PATCH 234/723] kernel: Add heap stats to profile header --- api/profile | 5 +++++ src/kernel/profile.cpp | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/api/profile b/api/profile index 8c2cdfa0c7..d731643894 100644 --- a/api/profile +++ b/api/profile @@ -150,4 +150,9 @@ class ScopedProfiler static std::array entries; }; +struct HeapDiag +{ + static std::string to_string(); +}; + #endif diff --git a/src/kernel/profile.cpp b/src/kernel/profile.cpp index bf61427684..508be6e110 100644 --- a/src/kernel/profile.cpp +++ b/src/kernel/profile.cpp @@ -197,3 +197,25 @@ void StackSampler::set_mask(bool mask) { get().discard = mask; } + +std::string HeapDiag::to_string() +{ + static intptr_t last = 0; + // show information on heap status, to discover leaks etc. + auto heap_begin = OS::heap_begin(); + auto heap_end = OS::heap_end(); + auto heap_usage = OS::heap_usage(); + intptr_t heap_size = heap_end - heap_begin; + last = heap_size - last; + + char buffer[256]; + int len = snprintf(buffer, sizeof(buffer), + "Heap begin %#lx size %lu Kb\n" + "Heap end %#lx diff %lu (%ld Kb)\n" + "Heap usage %lu kB\n", + heap_begin, heap_size / 1024, + heap_end, last, last / 1024, + heap_usage / 1024); + last = (int32_t) heap_size; + return std::string(buffer, len); +} From 285147b0be37c0fa179dea5c4a3c793c5cd60894 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 9 May 2018 14:57:18 +0200 Subject: [PATCH 235/723] net: Check if callback set in read request --- src/net/tcp/read_request.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/tcp/read_request.cpp b/src/net/tcp/read_request.cpp index bdaf8d8aa8..c6449dfa52 100644 --- a/src/net/tcp/read_request.cpp +++ b/src/net/tcp/read_request.cpp @@ -55,7 +55,7 @@ namespace tcp { while(buf->is_ready() and buf == buffers.front().get()) { const auto rem = buf->capacity() - buf->size(); - callback(buf->buffer()); + if (callback) callback(buf->buffer()); // this is the only one, so we can reuse it if(buffers.size() == 1) From bf32be09e604ecba3fe2438a9feb76731d8c09c3 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 9 May 2018 15:07:51 +0200 Subject: [PATCH 236/723] delegate: Add custom empty delegate exception --- api/util/delegate.hpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/util/delegate.hpp b/api/util/delegate.hpp index 6cc473de3d..d8d469cfed 100644 --- a/api/util/delegate.hpp +++ b/api/util/delegate.hpp @@ -57,6 +57,13 @@ template< > class delegate; +class empty_delegate_error : public std::bad_function_call +{ +public: + const char* what() const throw() { + return "Empty delegate called"; + } +}; // ----- IMPLEMENTATION ----- @@ -64,7 +71,7 @@ namespace detail { template static R empty_pure(Args...) { - throw std::bad_function_call(); + throw empty_delegate_error(); } template< From 21a25f1c759e3f75e32f855e0d7cebad77e81127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 9 May 2018 16:57:19 +0200 Subject: [PATCH 237/723] fs: allow to construct custom memdisk --- api/fs/memdisk.hpp | 1 + src/fs/memdisk.cpp | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/api/fs/memdisk.hpp b/api/fs/memdisk.hpp index cac315c940..8ac1c963d2 100644 --- a/api/fs/memdisk.hpp +++ b/api/fs/memdisk.hpp @@ -64,6 +64,7 @@ namespace fs { bool write_sync(block_t, buffer_t) override { return true; }; explicit MemDisk() noexcept; + explicit MemDisk(const char* start, const char* end) noexcept; void deactivate() override; diff --git a/src/fs/memdisk.cpp b/src/fs/memdisk.cpp index 941f81adc3..7d8fb28855 100644 --- a/src/fs/memdisk.cpp +++ b/src/fs/memdisk.cpp @@ -23,22 +23,26 @@ #include #include -extern "C" { - char _DISK_START_; - char _DISK_END_; -} + +extern char _DISK_START_; +extern char _DISK_END_; namespace fs { MemDisk::MemDisk() noexcept - : Block_device(), - image_start_ { &_DISK_START_ }, - image_end_ { &_DISK_END_ }, + : MemDisk(&_DISK_START_, &_DISK_END_) + { + INFO("Memdisk", "Initializing"); + } - stat_read( Statman::get().create( + MemDisk::MemDisk(const char* start, const char* end) noexcept + : Block_device(), + image_start_ { start }, + image_end_ { end }, + stat_read( Statman::get().create( Stat::UINT64, device_name() + ".reads").get_uint64() ) { - INFO("Memdisk", "Initializing"); + Expects(image_start_ <= image_end_); } MemDisk::buffer_t MemDisk::read_sync(block_t blk) From 3b80f1f9b06de3c5eaa1c9e3b16962d902fd5bcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 9 May 2018 17:00:42 +0200 Subject: [PATCH 238/723] gcc: changes for building unittests with GCC 8.1 on Mac --- api/hw/devices.hpp | 2 +- api/net/tcp/connection.hpp | 2 +- api/util/fixed_queue.hpp | 1 - src/include/kprint | 3 --- test/CMakeLists.txt | 6 +++--- test/fs/unit/memdisk_test.cpp | 7 ++++--- test/kernel/unit/memory.cpp | 4 ++-- test/lest_util/os_mock.cpp | 3 +++ test/net/unit/router_test.cpp | 14 +++++++++----- test/util/unit/bitops.cpp | 4 ++-- test/util/unit/lstack.cpp | 2 +- 11 files changed, 26 insertions(+), 22 deletions(-) diff --git a/api/hw/devices.hpp b/api/hw/devices.hpp index 3ea5132bc8..9e5c060d58 100644 --- a/api/hw/devices.hpp +++ b/api/hw/devices.hpp @@ -125,7 +125,7 @@ namespace hw { try { return *(devices().at(N)); } - catch(std::out_of_range) + catch(const std::out_of_range&) { throw Device_not_found{Device_type::device_type(), N}; } diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index f88fdbf8b4..70cc42d52b 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -1028,7 +1028,7 @@ class Connection { uint32_t usable_window() const noexcept { const auto x = (int64_t)send_window() - (int64_t)flight_size(); - return (uint32_t) std::max((decltype(x)) 0, x); + return (uint32_t) std::max(0ll, x); } uint32_t send_window() const noexcept diff --git a/api/util/fixed_queue.hpp b/api/util/fixed_queue.hpp index 2b0fb12900..28cfc3d819 100644 --- a/api/util/fixed_queue.hpp +++ b/api/util/fixed_queue.hpp @@ -60,7 +60,6 @@ template class fixed_queue T& back() noexcept { return buff_[(index_ + 1) % N]; } template void fold(F&& func) - noexcept(noexcept(func(front()))) { for (size_t i = 0, max = index_ < N ? index_ : N ; i < max; ++i) std::forward(func)(buff_[(index_ - i) % N]); diff --git a/src/include/kprint b/src/include/kprint index 3eceeab1b8..a3d0a2f378 100644 --- a/src/include/kprint +++ b/src/include/kprint @@ -40,9 +40,6 @@ extern void __serial_print(const char* str, size_t len); **/ __attribute__ ((format (printf, 1, 2))) -#ifdef __cplusplus -extern "C" -#endif void kprintf(const char* format, ...); extern void kprint(const char*); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index b328ea2559..4edf7341bb 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,7 +20,7 @@ add_definitions(-DARCH_${ARCH}) add_definitions(-DARCH="${ARCH}") add_definitions(-DOS_VERSION="v0.0.0.1") -set(CMAKE_C_FLAGS "-g -O0 -std=c11 -Wall -Wextra") +set(CMAKE_C_FLAGS "-g -O2 -std=c11 -Wall -Wextra") set(NO_INFO "-DNO_INFO=1") if(INFO) @@ -32,9 +32,9 @@ if (DEBUG_INFO) set(NO_DEBUG "") endif() -set(CMAKE_CXX_FLAGS "-g -O0 -march=native -std=c++17 -Wall -Wextra -Wno-unused-function -D__id_t_defined -DUNITTESTS -DURI_THROW_ON_ERROR ${NO_INFO} ${NO_DEBUG} -DGSL_THROW_ON_CONTRACT_VIOLATION -Dlest_FEATURE_AUTO_REGISTER=1 -DHAVE_LEST_MAIN") +set(CMAKE_CXX_FLAGS "-g -O2 -march=native -std=c++17 -Wall -Wextra -Wno-unused-function -D__id_t_defined -DUNITTESTS -DURI_THROW_ON_ERROR ${NO_INFO} ${NO_DEBUG} -DGSL_THROW_ON_CONTRACT_VIOLATION -Dlest_FEATURE_AUTO_REGISTER=1 -DHAVE_LEST_MAIN") -if (APPLE) +if (APPLE AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") message(STATUS "Including brew bundled libc++") execute_process(COMMAND brew "--prefix" "llvm@5" OUTPUT_VARIABLE BREW_LLVM) string(STRIP ${BREW_LLVM} BREW_LLVM) diff --git a/test/fs/unit/memdisk_test.cpp b/test/fs/unit/memdisk_test.cpp index 5c62597bdf..9a54e9633d 100644 --- a/test/fs/unit/memdisk_test.cpp +++ b/test/fs/unit/memdisk_test.cpp @@ -20,7 +20,8 @@ CASE("memdisk properties") { - fs::Disk& disk = fs::memdisk(); + fs::MemDisk memdisk{0,0}; + fs::Disk disk{memdisk}; EXPECT(disk.empty() == true); EXPECT(disk.device_id() == 0); EXPECT(disk.fs_ready() == false); @@ -34,9 +35,9 @@ CASE("memdisk properties") [&enumerated_partitions, &lest_env] (auto err, auto& partitions) { - EXPECT(!err); + EXPECT(err); enumerated_partitions = true; - EXPECT(partitions.size() == 4u); // 4 is default number + EXPECT(partitions.size() == 0u); // 4 is default number }); EXPECT(enumerated_partitions == true); } diff --git a/test/kernel/unit/memory.cpp b/test/kernel/unit/memory.cpp index ec388ed26c..a0baaec9ac 100644 --- a/test/kernel/unit/memory.cpp +++ b/test/kernel/unit/memory.cpp @@ -79,8 +79,8 @@ CASE("os::mem::Mapping Addition") n.lin = m.lin + m.size; auto old_m = m; auto old_n = n; - - EXPECT(m + n == (m += n)); + m += n; + EXPECT(old_m + old_n == m); EXPECT(m.size == old_m.size + old_n.size); EXPECT(m.lin == old_m.lin); diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index c706162091..56d0233f94 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -31,6 +31,9 @@ void* aligned_alloc(size_t alignment, size_t size) { } #endif +char _DISK_START_; +char _DISK_END_; + #include Statman& Statman::get() { static uintptr_t start {0}; diff --git a/test/net/unit/router_test.cpp b/test/net/unit/router_test.cpp index 8ebd0a6dcd..46f53a70fe 100644 --- a/test/net/unit/router_test.cpp +++ b/test/net/unit/router_test.cpp @@ -185,7 +185,7 @@ CASE("net::router: Actual routing verifying TTL") inet1.ip_obj().set_linklayer_out([&](auto pckt, auto ip) { auto packet = static_unique_ptr_cast(std::move(pckt)); EXPECT(packet->ip_protocol() == Protocol::ICMPv4); - EXPECT(packet->ip_ttl() == DEFAULT_TTL); + EXPECT(packet->ip_ttl() == PacketIP4::DEFAULT_TTL); auto icmp = icmp4::Packet(std::move(packet)); ICMP_error err{icmp.type(), icmp.code()}; @@ -201,7 +201,7 @@ CASE("net::router: Actual routing verifying TTL") auto packet = static_unique_ptr_cast(std::move(pckt)); EXPECT(packet->source() == src); EXPECT(packet->destination() == dst); - EXPECT(packet->ip_ttl() == DEFAULT_TTL-1); + EXPECT(packet->ip_ttl() == (PacketIP4::DEFAULT_TTL-1)); tcp_packet_recv++; }); @@ -229,7 +229,7 @@ CASE("net::router: Actual routing verifying TTL") // Ok this one is actual legit (default TTL) router.send_time_exceeded = true; tcp = create_tcp_packet_init(src, dst); - tcp->set_ip_ttl(DEFAULT_TTL); + tcp->set_ip_ttl(PacketIP4::DEFAULT_TTL); tcp->set_ip_checksum(); tcp->set_tcp_checksum(); inet1.ip_obj().receive(std::move(tcp), false); @@ -238,7 +238,9 @@ CASE("net::router: Actual routing verifying TTL") // Test the forward chain as well // Accept the first one - router.forward_chain.chain.push_back([](auto pkt, auto&, const auto*)->auto { + router.forward_chain.chain.push_back([]( + IP4::IP_packet_ptr pkt, Inet&, Conntrack::Entry_ptr)->Filter_verdict + { return Filter_verdict{std::move(pkt), Filter_verdict_type::ACCEPT}; }); @@ -249,7 +251,9 @@ CASE("net::router: Actual routing verifying TTL") EXPECT(tcp_packet_recv == 2); // Lets drop the next one - router.forward_chain.chain.push_back([](auto pkt, auto&, const auto*)->auto { + router.forward_chain.chain.push_back([]( + IP4::IP_packet_ptr pkt, Inet&, Conntrack::Entry_ptr)->Filter_verdict + { return Filter_verdict{std::move(pkt), Filter_verdict_type::DROP}; }); diff --git a/test/util/unit/bitops.cpp b/test/util/unit/bitops.cpp index f31b5d829a..dd50811cf1 100644 --- a/test/util/unit/bitops.cpp +++ b/test/util/unit/bitops.cpp @@ -37,7 +37,7 @@ using namespace util; // Enable bitmask ops for the Flags enum template<> -struct enable_bitmask_ops { +struct bitops::enable_bitmask_ops { using type = std::underlying_type::type; static constexpr bool enable = true; }; @@ -81,7 +81,7 @@ CASE ("util::bitops: Using bitmask ops for an enum") // Enable bitmask ops for int, to use in combination with other enabled types template<> -struct enable_bitmask_ops { +struct bitops::enable_bitmask_ops { using type = int; static constexpr bool enable = true; }; diff --git a/test/util/unit/lstack.cpp b/test/util/unit/lstack.cpp index 06ca829786..8740091be6 100644 --- a/test/util/unit/lstack.cpp +++ b/test/util/unit/lstack.cpp @@ -16,7 +16,7 @@ #include #include -#include +extern void* memalign(size_t alignment, size_t size); using Lstack = util::alloc::Lstack<>; using Chunk = Lstack::Chunk; From 6b700273e7c90ff4ac5c0eddee7f5bf58439ab03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 9 May 2018 17:02:00 +0200 Subject: [PATCH 239/723] util: Changes to Fixed alloc and storage to make it work with GCC --- api/util/fixed_list_alloc.hpp | 29 +++++++++++++++++++++++------ api/util/fixed_storage.hpp | 9 ++++++++- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/api/util/fixed_list_alloc.hpp b/api/util/fixed_list_alloc.hpp index 1bfd4d9ad1..0954c6c8a5 100644 --- a/api/util/fixed_list_alloc.hpp +++ b/api/util/fixed_list_alloc.hpp @@ -20,6 +20,7 @@ #define UTIL_FIXED_LIST_ALLOC_HPP #include "fixed_storage.hpp" +#include // Implemenation based upon Howard Hinnant's terrific short_alloc example // https://howardhinnant.github.io/short_alloc.h @@ -46,31 +47,47 @@ class Fixed_list_alloc { "Total size (sizeof(T) * N) needs to be a multiple of alignment Align"); private: - storage_type store_; + std::unique_ptr store_; public: - Fixed_list_alloc() noexcept - : store_() {} + Fixed_list_alloc() + : store_{std::make_unique()} + {} + template + Fixed_list_alloc(const U&) + : store_{std::make_unique()} + { + // ain't much to do here since storage is fixed to T, + // so we just create a new storage for this type + } + + Fixed_list_alloc(Fixed_list_alloc&&) noexcept = default; + Fixed_list_alloc& operator=(Fixed_list_alloc&&) noexcept = default; + + /* No copy */ Fixed_list_alloc(const Fixed_list_alloc&) = delete; Fixed_list_alloc& operator=(const Fixed_list_alloc&) = delete; + template + using other = Fixed_list_alloc; + template struct rebind { using other = Fixed_list_alloc; }; T* allocate(std::size_t n) { - return reinterpret_cast(store_.template allocate(n*sizeof(T))); + return reinterpret_cast(store_->template allocate(n*sizeof(T))); } void deallocate(T* p, std::size_t n) noexcept { - store_.deallocate(reinterpret_cast(p), n*sizeof(T)); + store_->deallocate(reinterpret_cast(p), n*sizeof(T)); } bool operator==(const Fixed_list_alloc& other) const noexcept - { return this == &other; } + { return this->store_ == other.store_; } bool operator!=(const Fixed_list_alloc& other) const noexcept { return !(*this == other); } diff --git a/api/util/fixed_storage.hpp b/api/util/fixed_storage.hpp index 1d9c97844b..2524cc6e18 100644 --- a/api/util/fixed_storage.hpp +++ b/api/util/fixed_storage.hpp @@ -48,7 +48,6 @@ class Fixed_storage { static constexpr std::size_t buffer_size() noexcept { return N * aligned_size(); } - private: alignas(alignment) std::array buf_; /** Available addresses */ @@ -57,6 +56,11 @@ class Fixed_storage { public: Fixed_storage() noexcept; + Fixed_storage(const Fixed_storage&) = delete; + Fixed_storage(Fixed_storage&&) = delete; + Fixed_storage& operator=(const Fixed_storage&) = delete; + Fixed_storage& operator=(Fixed_storage&&) = delete; + template char* allocate(std::size_t n); void deallocate(char* p, std::size_t n) noexcept; @@ -108,6 +112,9 @@ char* Fixed_storage::allocate(std::size_t n) template void Fixed_storage::deallocate(char* p, std::size_t) noexcept { + //printf("Fixed_storage<%s, %u, %u> dealloc %p\n", + // typeid(T).name(), N, alignment, p); + Expects(pointer_in_buffer(p) && "Trying to deallocate pointer outside my buffer"); From 96b5d7c4aafab77ba4f4b49c845dcb25be1c987f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 9 May 2018 17:17:54 +0200 Subject: [PATCH 240/723] tcp: specifically tell what types to use when calc send wnd --- api/net/tcp/connection.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index 70cc42d52b..f0d1fe826e 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -1027,8 +1027,8 @@ class Connection { */ uint32_t usable_window() const noexcept { - const auto x = (int64_t)send_window() - (int64_t)flight_size(); - return (uint32_t) std::max(0ll, x); + const int64_t x = (int64_t)send_window() - (int64_t)flight_size(); + return (uint32_t) std::max(static_cast(0), x); } uint32_t send_window() const noexcept From aeb28cda0d2bf5ff197571190e62e27ccb555ce0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 9 May 2018 17:48:23 +0200 Subject: [PATCH 241/723] test: Router test lambdas updated to new Filter signature --- test/net/unit/router_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/net/unit/router_test.cpp b/test/net/unit/router_test.cpp index aa4536a511..4fa78ee5ee 100644 --- a/test/net/unit/router_test.cpp +++ b/test/net/unit/router_test.cpp @@ -239,7 +239,7 @@ CASE("net::router: Actual routing verifying TTL") // Test the forward chain as well // Accept the first one router.forward_chain.chain.push_back([]( - IP4::IP_packet_ptr pkt, Inet&, Conntrack::Entry_ptr)->Filter_verdict + IP4::IP_packet_ptr pkt, Inet&, Conntrack::Entry_ptr)->Filter_verdict { return Filter_verdict{std::move(pkt), Filter_verdict_type::ACCEPT}; }); @@ -252,7 +252,7 @@ CASE("net::router: Actual routing verifying TTL") // Lets drop the next one router.forward_chain.chain.push_back([]( - IP4::IP_packet_ptr pkt, Inet&, Conntrack::Entry_ptr)->Filter_verdict + IP4::IP_packet_ptr pkt, Inet&, Conntrack::Entry_ptr)->Filter_verdict { return Filter_verdict{std::move(pkt), Filter_verdict_type::DROP}; }); From 65e30f5247e80ff4fe993078bead97b0f43ec55a Mon Sep 17 00:00:00 2001 From: niks3089 Date: Fri, 27 Apr 2018 10:31:00 +0530 Subject: [PATCH 242/723] ip6: Initial icmpv6 support and very basic NDP supprt along with tests --- api/net/inet.hpp | 38 +- api/net/ip6/addr.hpp | 83 ++- api/net/ip6/header.hpp | 9 +- api/net/ip6/icmp6.hpp | 254 ++++---- api/net/ip6/icmp6_common.hpp | 159 +++++ api/net/ip6/icmp6_error.hpp | 99 +++ api/net/ip6/ip6.hpp | 5 +- api/net/ip6/ndp.hpp | 72 -- api/net/ip6/packet_icmp6.hpp | 273 ++++++++ api/net/ip6/packet_ip6.hpp | 55 +- etc/scripts/create_bridge.sh | 4 + src/CMakeLists.txt | 2 +- src/net/inet.cpp | 47 +- src/net/ip6/icmp6.cpp | 616 ++++++++++-------- src/net/ip6/ip6.cpp | 52 +- src/net/ip6/ndp.cpp | 2 +- test/CMakeLists.txt | 3 + test/lest_util/packet_factory.hpp | 19 + .../{ipv6 => icmp6}/CMakeLists.txt | 17 +- .../integration/{ipv6 => icmp6}/service.cpp | 40 +- test/net/integration/{ipv6 => icmp6}/vm.json | 2 +- test/net/unit/ip6.cpp | 2 +- test/net/unit/ip6_addr.cpp | 74 +++ test/net/unit/ip6_packet_test.cpp | 66 ++ 24 files changed, 1426 insertions(+), 567 deletions(-) create mode 100644 api/net/ip6/icmp6_common.hpp create mode 100644 api/net/ip6/icmp6_error.hpp delete mode 100644 api/net/ip6/ndp.hpp create mode 100644 api/net/ip6/packet_icmp6.hpp rename test/net/integration/{ipv6 => icmp6}/CMakeLists.txt (71%) rename test/net/integration/{ipv6 => icmp6}/service.cpp (50%) rename test/net/integration/{ipv6 => icmp6}/vm.json (78%) create mode 100644 test/net/unit/ip6_addr.cpp create mode 100644 test/net/unit/ip6_packet_test.cpp diff --git a/api/net/inet.hpp b/api/net/inet.hpp index e7f8731b99..8c9de420fa 100644 --- a/api/net/inet.hpp +++ b/api/net/inet.hpp @@ -33,30 +33,28 @@ #include "ip4/icmp4.hpp" #include "ip4/arp.hpp" #include "ip6/ip6.hpp" +#include "ip6/icmp6.hpp" #include "dns/client.hpp" #include "tcp/tcp.hpp" #include "super_stack.hpp" namespace net { - class Arp; - class TCP; - class UDP; - class ICMPv4; class DHClient; - class Super_stack; /** A complete IP network stack */ class Inet { public: - using Stack = class Inet; - using IP_packet_ptr = IP4::IP_packet_ptr; - using IP_addr = IP4::addr; + using Stack = class Inet; + using IP_packet_ptr = IP4::IP_packet_ptr; + using IP6_packet_ptr = IP6::IP_packet_ptr; + using IP_addr = IP4::addr; using Forward_delg = delegate; using Route_checker = delegate; - using IP_packet_factory = delegate; + using IP_packet_factory = delegate; + using IP6_packet_factory = delegate; using resolve_func = delegate; using on_configured_func = delegate; @@ -93,7 +91,7 @@ namespace net { IP6::addr ip6_addr() const { return ip6_addr_; } - IP6::addr netmask6() const + uint8_t netmask6() const { return ip6_prefix_; } IP6::addr gateway6() const @@ -120,6 +118,9 @@ namespace net { /** Get the ICMP-object belonging to this stack */ ICMPv4& icmp() { return icmp_; } + /** Get the ICMP-object belonging to this stack */ + ICMPv6& icmp6() { return icmp6_; } + /** Get the DHCP client (if any) */ auto dhclient() { return dhcp_; } @@ -210,6 +211,9 @@ namespace net { IP_packet_factory ip_packet_factory() { return IP_packet_factory{this, &Inet::create_ip_packet}; } + IP6_packet_factory ip6_packet_factory() + { return IP6_packet_factory{this, &Inet::create_ip6_packet}; } + /** MTU retreived from Nic on construction */ uint16_t MTU () const { return MTU_; } @@ -279,9 +283,10 @@ namespace net { void network_config(IP4::addr addr, IP4::addr nmask, IP4::addr gateway, - IP4::addr dns = IP4::ADDR_ANY, - IP6::addr addr6 = IP6::ADDR_ANY, - IP6::addr prefix6 = IP6::ADDR_ANY, + IP4::addr dns = IP4::ADDR_ANY); + + void network_config6(IP6::addr addr6 = IP6::ADDR_ANY, + uint8_t prefix6 = 0, IP6::addr gateway6 = IP6::ADDR_ANY); void @@ -292,7 +297,7 @@ namespace net { this->netmask_ = IP4::ADDR_ANY; this->ip6_addr_ = IP6::ADDR_ANY; this->ip6_gateway_ = IP6::ADDR_ANY; - this->ip6_prefix_ = IP6::ADDR_ANY; + this->ip6_prefix_ = 0; } // register a callback for receiving signal on free packet-buffers @@ -428,7 +433,7 @@ namespace net { { return src == ip_addr() or is_loopback(src); } bool is_valid_source(IP6::addr src) - { return src == ip6_addr() or is_loopback(src); } + { return src == ip6_addr() or is_loopback(src) or src.is_multicast(); } std::shared_ptr& conntrack() { return conntrack_; } @@ -457,7 +462,7 @@ namespace net { IP6::addr ip6_addr_; IP6::addr ip6_gateway_; - IP6::addr ip6_prefix_; + uint8_t ip6_prefix_; Vip4_list vip4s_ = {{127,0,0,1}}; Vip6_list vip6s_ = {{IP6::ADDR_LOOPBACK}}; @@ -468,6 +473,7 @@ namespace net { IP4 ip4_; IP6 ip6_; ICMPv4 icmp_; + ICMPv6 icmp6_; UDP udp_; TCP tcp_; diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index 8bca66a7d8..41d8501428 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -46,16 +46,16 @@ struct Addr { Addr(uint16_t a1, uint16_t a2, uint16_t b1, uint16_t b2, uint16_t c1, uint16_t c2, uint16_t d1, uint16_t d2) { - i16[0] = htons(a1); i16[1] = htons(a2); - i16[2] = htons(b1); i16[3] = htons(b2); - i16[4] = htons(c1); i16[5] = htons(c2); - i16[6] = htons(d1); i16[7] = htons(d2); + i16[7] = a1; i16[6] = a2; + i16[5] = b1; i16[4] = b2; + i16[3] = c1; i16[2] = c2; + i16[1] = d1; i16[0] = d2; } Addr(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - i32[0] = htonl(a); i32[1] = htonl(b); - i32[2] = htonl(c); i32[3] = htonl(d); + i32[0] = d; i32[1] = c; + i32[2] = b; i32[3] = a; } Addr(const Addr& a) @@ -70,9 +70,9 @@ struct Addr { char ipv6_addr[32]; snprintf(ipv6_addr, sizeof(ipv6_addr), "%0x:%0x:%0x:%0x:%0x:%0x:%0x:%0x", - ntohs(i16[0]), ntohs(i16[1]), ntohs(i16[2]), - ntohs(i16[3]), ntohs(i16[4]), ntohs(i16[5]), - ntohs(i16[6]), ntohs(i16[7])); + i16[7], i16[6], i16[5], + i16[4], i16[3], i16[2], + i16[1], i16[0]); return ipv6_addr; } @@ -152,35 +152,40 @@ struct Addr { { return not (*this == other); } /** - * Operator to check for less-than relationship + * Operator to check for greater-than relationship */ - bool operator<(const Addr other) const noexcept - { return i32[3] < other.i32[3] && - i32[2] < other.i32[2] && - i32[1] < other.i32[1] && - i32[0] < other.i32[0]; } + bool operator>(const Addr other) const noexcept + { + if (i32[3] > other.i32[3]) return true; + if (i32[2] > other.i32[2]) return true; + if (i32[1] > other.i32[1]) return true; + if (i32[0] > other.i32[0]) return true; + + return false; + } + /** - * Operator to check for less-than-or-equal relationship + * Operator to check for greater-than-or-equal relationship */ - bool operator<=(const Addr other) const noexcept - { return (*this < other or *this == other); } + bool operator>=(const Addr other) const noexcept + { return (*this > other or *this == other); } /** - * Operator to check for greater-than relationship + * Operator to check for lesser-than relationship */ - bool operator>(const Addr other) const noexcept - { return not (*this <= other); } + bool operator<(const Addr other) const noexcept + { return not (*this >= other); } /** - * Operator to check for greater-than-or-equal relationship + * Operator to check for lesser-than-or-equal relationship */ - bool operator>=(const Addr other) const noexcept - { return (*this > other or *this == other); } + bool operator<=(const Addr other) const noexcept + { return (*this < other or *this == other); } /** * Operator to perform a bitwise-and operation on the given - * IPv4 addresses + * IPv6 addresses */ Addr operator&(const Addr other) const noexcept { return Addr{i32[3] & other.i32[3], @@ -188,6 +193,34 @@ struct Addr { i32[1] & other.i32[1], i32[0] & other.i32[0] }; } + Addr operator&(uint8_t prefix) const noexcept + { + int i = 0; + uint8_t mask; + uint32_t addr[32]; + + addr[0] = i32[0]; + addr[1] = i32[1]; + addr[2] = i32[2]; + addr[3] = i32[3]; + + if (prefix > 128) { + prefix = 128; + } + + mask = 128 - prefix; + while (mask >= 32) { + addr[i++] = 0; + mask -= 32; + } + + if (mask != 0) { + addr[i] &= (0xFFFFFFFF << mask); + } + return Addr { addr[3], addr[2], + addr[1], addr[0] }; + } + Addr operator|(const Addr other) const noexcept { return Addr{i32[3] | other.i32[3], i32[2] | other.i32[2], diff --git a/api/net/ip6/header.hpp b/api/net/ip6/header.hpp index 8e8e198cd2..69f37b0965 100644 --- a/api/net/ip6/header.hpp +++ b/api/net/ip6/header.hpp @@ -30,9 +30,12 @@ namespace ip6 { * This type is used to represent the standard IPv6 header */ struct Header { - uint32_t version:4, - traffic_class:8, - flow_label:20; + union { + uint32_t ver_tc_fl; + uint32_t version : 4, + traffic_class : 8, + flow_label : 20; + }; uint16_t payload_length; uint8_t next_header; uint8_t hop_limit; diff --git a/api/net/ip6/icmp6.hpp b/api/net/ip6/icmp6.hpp index 858b9122c2..c779d33d06 100644 --- a/api/net/ip6/icmp6.hpp +++ b/api/net/ip6/icmp6.hpp @@ -20,24 +20,84 @@ #ifndef NET_IP6_ICMPv6_HPP #define NET_IP6_ICMPv6_HPP -//#include "packet_icmp6.hpp" +#include "packet_icmp6.hpp" #include #include namespace net { - class PacketICMP6; + /** + * User friendly ICMP packet (view) used in ping callback (icmp_func) + */ + class ICMP6_view { + + using ICMP_type = ICMP6_error::ICMP_type; + using ICMP_code = ICMP6_error::ICMP_code; + + public: + ICMP6_view() {} + + ICMP6_view(icmp6::Packet& pckt) + : id_{pckt.id()}, + seq_{pckt.sequence()}, + src_{pckt.ip().ip_src()}, + dst_{pckt.ip().ip_dst()}, + type_{pckt.type()}, + code_{pckt.code()}, + checksum_{pckt.checksum()}, + payload_{(const char*) pckt.payload().data(), (size_t) pckt.payload().size()} + {} + + uint16_t id() const noexcept + { return id_; } + + uint16_t seq() const noexcept + { return seq_; } + + IP6::addr src() const noexcept + { return src_; } + + IP6::addr dst() const noexcept + { return dst_; } + + ICMP_type type() const noexcept + { return type_; } + + ICMP_code code() const noexcept + { return code_; } + + uint16_t checksum() const noexcept + { return checksum_; } + + std::string payload() const + { return payload_; } + + operator bool() const noexcept + { return type_ != ICMP_type::NO_REPLY; } + + std::string to_string() const; + + private: + uint16_t id_{0}; + uint16_t seq_{0}; + IP6::addr src_{0,0,0,0}; + IP6::addr dst_{0,0,0,0}; + ICMP_type type_{ICMP_type::NO_REPLY}; + uint8_t code_{0}; + uint16_t checksum_{0}; + std::string payload_{""}; + + }; // < class ICMP6_view class ICMPv6 { -#if 0 - using ICMP_type = ICMP_error::ICMP_type; - using ICMP_code = ICMP_error::ICMP_code; + using ICMP_type = ICMP6_error::ICMP_type; + using ICMP_code = ICMP6_error::ICMP_code; public: using Stack = IP6::Stack; using Tuple = std::pair; // identifier and sequence number - using icmp_func = delegate; + using icmp_func = delegate; static const int SEC_WAIT_FOR_REPLY = 40; @@ -59,7 +119,7 @@ namespace net /** * */ - void redirect(Packet_ptr pckt, icmp6::code::Redirect code); + //void redirect(Packet_ptr pckt, icmp6::code::Redirect code); /** * Sending a Time Exceeded message from a host when fragment reassembly time exceeded (code 1) @@ -77,7 +137,6 @@ namespace net */ void parameter_problem(Packet_ptr pckt, uint8_t error_pointer); - // May void timestamp_request(IP6::addr ip); void timestamp_reply(icmp6::Packet& req); @@ -86,118 +145,95 @@ namespace net void ping(const std::string& hostname); void ping(const std::string& hostname, icmp_func callback, int sec_wait = SEC_WAIT_FOR_REPLY); -#endif - -#if 0 - typedef uint8_t type_t; - typedef int (*handler_t)(ICMPv6&, std::shared_ptr&); - - ICMPv6(IP6::addr& ip6); - - struct header - { - uint8_t type; - uint8_t code; - uint16_t checksum; - } __attribute__((packed)); - - struct pseudo_header - { - IP6::addr src; - IP6::addr dst; - uint32_t len; - uint8_t zeros[3]; - uint8_t next; - } __attribute__((packed)); - - struct echo_header - { - uint8_t type; - uint8_t code; - uint16_t checksum; - uint16_t identifier; - uint16_t sequence; - uint8_t data[0]; - } __attribute__((packed)); - - // packet from IP6 layer - int bottom(Packet_ptr pckt); - - // set the downstream delegate - inline void set_ip6_out(IP6::downstream6 del) - { - this->ip6_out = del; - } + private: + static int request_id_; // message identifier for messages originating from IncludeOS + Stack& inet_; + downstream network_layer_out_ = nullptr; + uint8_t includeos_payload_[48] = {'I','N','C','L','U','D', + 'E','O','S','1','2','3','4','5', + 'A','B','C','D','E','F','G','H', + 'I','J','K','L','M','N','O','P', + 'Q','R','S','T','U','V','W','X', + 'Y','Z','1','2','3','4','5','6', + '7','8'}; + + inline bool is_full_header(size_t pckt_size) + { return (pckt_size >= sizeof(IP6::header) + icmp6::Packet::header_size()); } + + struct ICMP_callback { + using icmp_func = ICMPv6::icmp_func; + using Tuple = ICMPv6::Tuple; + + Tuple tuple; + icmp_func callback; + Timers::id_t timer_id; + + ICMP_callback(ICMPv6& icmp, Tuple t, icmp_func cb, int sec_wait) + : tuple{t}, callback{cb} + { + timer_id = Timers::oneshot(std::chrono::seconds(sec_wait), [&icmp, t](Timers::id_t) { + icmp.remove_ping_callback(t); + }); + } + }; // < struct ICMP_callback + + std::map ping_callbacks_; + + void forward_to_transport_layer(icmp6::Packet& req); - inline const IP6::addr& local_ip() - { - return localIP; - } + /** + * If a Destination Unreachable: Fragmentation Needed error mesage is received, the Path MTUs in + * IP should be updated and the packetization/transportation layer should be notified (through Inet) + * RFC 1191 (Path MTU Discovery IP4), 1981 (Path MTU Discovery IP6) and 4821 (Packetization Layer Path MTU Discovery) + * A Fragmentation Needed error message is sent as a response to a packet with an MTU that is too big + * for a node in the path to its destination and with the Don't Fragment bit set as well + */ + void handle_too_big(icmp6::Packet& req); - // message types & codes - static inline bool is_error(uint8_t type) - { - return type < 128; + /** + * Find the ping-callback that this packet is a response to, execute it and erase the object + * from the ping_callbacks_ map + */ + inline void execute_ping_callback(icmp6::Packet& ping_response) { + // Find callback matching the reply + auto it = ping_callbacks_.find(std::make_pair(ping_response.id(), ping_response.sequence())); + + if (it != ping_callbacks_.end()) { + it->second.callback(ICMP6_view{ping_response}); + Timers::stop(it->second.timer_id); + ping_callbacks_.erase(it); + } } - static std::string code_string(uint8_t type, uint8_t code); - // calculate checksum of any ICMP message - static uint16_t checksum(std::shared_ptr& pckt); + /** Remove ICMP_callback from ping_callbacks_ map when its timer timeouts */ + inline void remove_ping_callback(Tuple key) { + auto it = ping_callbacks_.find(key); - // provide a handler for a @type of ICMPv6 message - inline void listen(type_t type, handler_t func) - { - listeners[type] = func; + if (it != ping_callbacks_.end()) { + // Data back to user if no response found + it->second.callback(ICMP6_view{}); + Timers::stop(it->second.timer_id); + ping_callbacks_.erase(it); + } } - // transmit packet downstream - int transmit(std::shared_ptr& pckt); + void send_request(IP6::addr dest_ip, ICMP_type type, ICMP_code code, + icmp_func callback = nullptr, int sec_wait = SEC_WAIT_FOR_REPLY, uint16_t sequence = 0); - // send NDP router solicitation - void discover(); - - private: - std::map listeners; - // connection to IP6 layer - IP6::downstream6 ip6_out; - // this network stacks IPv6 address - IP6::addr& localIP; - }; - - class PacketICMP6 : public PacketIP6 - { - public: - inline ICMPv6::header& header() - { - return *(ICMPv6::header*) this->payload(); - } - inline const ICMPv6::header& header() const - { - return *(ICMPv6::header*) this->payload(); - } + /** Send response without id and sequence number */ + void send_response(icmp6::Packet& req, ICMP_type type, ICMP_code code, + uint8_t error_pointer = std::numeric_limits::max()); - inline uint8_t type() const - { - return header().type; - } - inline uint8_t code() const - { - return header().code; - } - inline uint16_t checksum() const - { - return ntohs(header().checksum); - } + /** + * Responding to a ping (echo) request + * Called from receive-method + */ + void ping_reply(icmp6::Packet&); - void set_length(uint32_t icmp_len) - { - // new total IPv6 payload length - ip6_header().set_size(icmp_len); - // new total packet length - set_size(sizeof(IP6::full_header) + icmp_len); - } -#endif + /* Receive neighbor solicitaion and respond with neighbor advertisement */ + int receive_neighbor_solicitation(icmp6::Packet&); + void send_router_solicitation(); }; } #endif diff --git a/api/net/ip6/icmp6_common.hpp b/api/net/ip6/icmp6_common.hpp new file mode 100644 index 0000000000..b5adda1078 --- /dev/null +++ b/api/net/ip6/icmp6_common.hpp @@ -0,0 +1,159 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +#pragma once +#ifndef NET_IP6_ICMP6_COMMON_HPP +#define NET_IP6_ICMP6_COMMON_HPP + +namespace net { + + /** ICMP6 Codes so that UDP and IP6 can use these */ + namespace icmp6 { + + // ICMP types + enum class Type : uint8_t { + DEST_UNREACHABLE = 1, + PACKET_TOO_BIG = 2, + TIME_EXCEEDED = 3, + PARAMETER_PROBLEM = 4, + ECHO = 128, + ECHO_REPLY = 129, + MULTICAST_LISTENER_QUERY = 130, + MULTICAST_LISTENER_REPORT = 131, + MULTICAST_LISTENER_DONE = 132, + ND_ROUTER_SOLICATION = 133, + ND_ROUTER_ADV = 134, + ND_NEIGHBOR_SOL = 135, + ND_NEIGHBOR_ADV = 136, + ND_REDIRECT = 137, + ROUTER_RENUMBERING = 138, + INFORMATION_QUERY = 139, + INFORMATION_RESPONSE = 140, + NO_REPLY = 199, // Custom: Type in ICMP_view if no ping reply received + NO_ERROR = 200 + }; + namespace code { + + enum class Dest_unreachable : uint8_t { + NO_ROUTE, + DEST_ADMIN_PROHIBIT, + NO_SRC_SCOPE, + ADDR_UNREACHABLE, + PORT_UNREACHABLE, + SRC_POLICY_FAIL, + REJECT_ROUTE_DEST, + SRC_ROUTING_ERR, + }; + + enum class Time_exceeded : uint8_t { + HOP_LIMIT, + FRAGMENT_REASSEMBLY + }; + + enum class Parameter_problem : uint8_t { + HEADER_ERR, + UNKNOWN_HEADER, + UNKNOWN_IPV6_OPT, + }; + } // < namespace code + + static std::string __attribute__((unused)) get_type_string(Type type) { + switch (type) { + case Type::DEST_UNREACHABLE: + return "DESTINATION UNREACHABLE (1)"; + case Type::PACKET_TOO_BIG: + return "PACKET TOO BIG (2)"; + case Type::TIME_EXCEEDED: + return "TIME EXCEEDED (3)"; + case Type::PARAMETER_PROBLEM: + return "PARAMETER PROBLEM (4)"; + case Type::ECHO: + return "ECHO (128)"; + case Type::ECHO_REPLY: + return "ECHO REPLY (129)"; + default: + return "UNKNOWN"; + } + } + + static std::string __attribute__((unused)) get_code_string(Type type, uint8_t code) { + switch (type) { + case Type::PACKET_TOO_BIG: // Have only code 0 + case Type::ECHO: + case Type::ECHO_REPLY: + case Type::MULTICAST_LISTENER_QUERY: + case Type::MULTICAST_LISTENER_REPORT: + case Type::MULTICAST_LISTENER_DONE: + case Type::ND_ROUTER_SOLICATION: + case Type::ND_ROUTER_ADV: + case Type::ND_NEIGHBOR_SOL: + case Type::ND_NEIGHBOR_ADV: + case Type::ND_REDIRECT: + case Type::ROUTER_RENUMBERING: + case Type::INFORMATION_QUERY: + case Type::INFORMATION_RESPONSE: + return "DEFAULT (0)"; + case Type::DEST_UNREACHABLE: + switch ( (code::Dest_unreachable) code ) { + case code::Dest_unreachable::NO_ROUTE: + return "NO ROUTE(0)"; + case code::Dest_unreachable::DEST_ADMIN_PROHIBIT: + return "DESTINATION ADMIN PROHIBITED (1)"; + case code::Dest_unreachable::NO_SRC_SCOPE: + return "SOURCE NO SCOPE(2)"; + case code::Dest_unreachable::ADDR_UNREACHABLE: + return "ADDR UNREACHABLE(3)"; + case code::Dest_unreachable::PORT_UNREACHABLE: + return "PORT UNREACHABLE(4)"; + case code::Dest_unreachable::SRC_POLICY_FAIL: + return "SOURCE POLICY FAILURE(5)"; + case code::Dest_unreachable::REJECT_ROUTE_DEST: + return "ROUTE DESTINATION REJECTED(6)"; + case code::Dest_unreachable::SRC_ROUTING_ERR: + return "SOURCE ROUTING ERROR(7)"; + default: + return "-"; + } + case Type::TIME_EXCEEDED: + switch ( (code::Time_exceeded) code ) { + case code::Time_exceeded::HOP_LIMIT: + return "HOP LIMIT(0)"; + case code::Time_exceeded::FRAGMENT_REASSEMBLY: + return "FRAGMENT REASSEMBLY (1)"; + default: + return "-"; + } + case Type::PARAMETER_PROBLEM: + switch ( (code::Parameter_problem) code ) { + case code::Parameter_problem::HEADER_ERR: + return "HEADER ERROR (0)"; + case code::Parameter_problem::UNKNOWN_HEADER: + return "UNKNOWN HEADER (1)"; + case code::Parameter_problem::UNKNOWN_IPV6_OPT: + return "UNKNOWN IP6 OPTION(2)"; + default: + return "-"; + } + default: + return "-"; + } + } + + } // < namespace icmp6 +} // < namespace net + +#endif diff --git a/api/net/ip6/icmp6_error.hpp b/api/net/ip6/icmp6_error.hpp new file mode 100644 index 0000000000..05e34e0398 --- /dev/null +++ b/api/net/ip6/icmp6_error.hpp @@ -0,0 +1,99 @@ + +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NET_IP6_ICMP_ERROR_HPP +#define NET_IP6_ICMP_ERROR_HPP + +#include +#include + +namespace net { + +/** + * An object of this error class is sent to UDP and TCP (via Inet) when an ICMP error message + * is received in ICMPv6::receive + */ + class ICMP6_error : public Error { + + public: + using ICMP_type = icmp6::Type; + using ICMP_code = uint8_t; + + /** + * @brief Constructor + * Default: No error occurred + */ + ICMP6_error() + : Error{} + {} + + /** + * @brief Constructor + * + * @param[in] icmp_type The ICMP type + * @param[in] icmp_code The ICMP code + * @param[in] pmtu The Path MTU (Maximum Transmission Unit for the destination) + * This is set in Inet, which asks the IP layer for the most recent + * Path MTU value + */ + ICMP6_error(ICMP_type icmp_type, ICMP_code icmp_code, uint16_t pmtu = 0) + : Error{Error::Type::ICMP, "ICMP error message received"}, + icmp_type_{icmp_type}, icmp_code_{icmp_code}, pmtu_{pmtu} + {} + + ICMP_type icmp_type() const noexcept + { return icmp_type_; } + + std::string icmp_type_str() const + { return icmp6::get_type_string(icmp_type_); } + + void set_icmp_type(ICMP_type icmp_type) noexcept + { icmp_type_ = icmp_type; } + + ICMP_code icmp_code() const noexcept + { return icmp_code_; } + + std::string icmp_code_str() const + { return icmp6::get_code_string(icmp_type_, icmp_code_); } + + void set_icmp_code(ICMP_code icmp_code) noexcept + { icmp_code_ = icmp_code; } + + bool is_too_big() const noexcept { + return icmp_type_ == ICMP_type::PACKET_TOO_BIG; + } + + uint16_t pmtu() const noexcept + { return pmtu_; } + + void set_pmtu(uint16_t pmtu) noexcept + { pmtu_ = pmtu; } + + std::string to_string() const override + { return "ICMP " + icmp_type_str() + ": " + icmp_code_str(); } + + private: + ICMP_type icmp_type_{ICMP_type::NO_ERROR}; + ICMP_code icmp_code_{0}; + uint16_t pmtu_{0}; // Is set if packet sent received an ICMP too big message + + }; // < class ICMP6_error + +} //< namespace net + +#endif //< NET_IP6_ICMP_ERROR_HPP diff --git a/api/net/ip6/ip6.hpp b/api/net/ip6/ip6.hpp index 083199fdbf..dbf650a6d5 100644 --- a/api/net/ip6/ip6.hpp +++ b/api/net/ip6/ip6.hpp @@ -47,6 +47,7 @@ namespace net using header = ip6::Header; using IP_packet = PacketIP6; using IP_packet_ptr = std::unique_ptr; + using IP_packet_factory = delegate; using downstream_ndp = delegate; using drop_handler = delegate; using Forward_delg = delegate; @@ -117,7 +118,7 @@ namespace net * * Destination IP * * Protocol * - * Source IP *can* be set - if it's not, IP4 will set it + * Source IP *can* be set - if it's not, IP6 will set it */ void transmit(Packet_ptr); void ship(Packet_ptr, addr next_hop = IP6::ADDR_ANY, Conntrack::Entry_ptr ct = nullptr); @@ -137,7 +138,7 @@ namespace net * @return True if for me, False otherwise. */ bool is_for_me(ip6::Addr dst) const; - void ipv6_ext_header_receive(Packet_ptr pckt); + Protocol ipv6_ext_header_receive(net::PacketIP6& packet); /// /// PACKET FILTERING diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp deleted file mode 100644 index 5ca2fd1b49..0000000000 --- a/api/net/ip6/ndp.hpp +++ /dev/null @@ -1,72 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once - -#include -#include "ip6.hpp" - -namespace net -{ - class NDP - { - public: - struct router_sol - { - uint8_t type; - uint8_t code; - uint16_t checksum; - uint32_t reserved; - uint8_t options[0]; - - } __attribute__((packed)); - - struct neighbor_sol - { - uint8_t type; - uint8_t code; - uint16_t checksum; - uint32_t reserved; - IP6::addr target; - uint8_t options[0]; - - } __attribute__((packed)); - - struct neighbor_adv - { - uint8_t type; - uint8_t code; - uint16_t checksum; - uint32_t rso_reserved; - IP6::addr target; - uint8_t options[0]; - - static const uint32_t R = 0x1; - static const uint32_t S = 0x2; - static const uint32_t O = 0x4; - - void set_rso(uint32_t rso_flags) - { - rso_reserved = htonl(rso_flags & 7); - } - - } __attribute__((packed)); - - - - }; -} diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp new file mode 100644 index 0000000000..fc3bd78db2 --- /dev/null +++ b/api/net/ip6/packet_icmp6.hpp @@ -0,0 +1,273 @@ + +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +namespace net { + +#define NEIGH_ADV_ROUTER 0x1 +#define NEIGH_ADV_SOL 0x2 +#define NEIGH_ADV_OVERRIDE 0x4 + +namespace icmp6 { + + class Packet { + struct Header { + Type type; + uint8_t code; + uint16_t checksum; + union { + uint16_t identifier; + uint16_t sequence; + uint32_t reserved; + uint32_t rso_flags; + }; + uint8_t payload[0]; + }__attribute__((packed)); + + Header& header() + { return *reinterpret_cast(pckt_->layer_begin() + pckt_->ip_header_len()); } + + const Header& header() const + { return *reinterpret_cast(pckt_->layer_begin() + pckt_->ip_header_len()); } + + struct pseudo_header + { + IP6::addr src; + IP6::addr dst; + uint32_t len; + uint8_t zeros[3]; + uint8_t next; + } __attribute__((packed)); + + struct RouterSol + { + uint8_t options[0]; + } __attribute__((packed)); + + struct RouterAdv + { + uint32_t reachable_time; + uint32_t retrans_timer; + } __attribute__((packed)); + + struct RouterRedirect + { + IP6::addr target; + IP6::addr dest; + uint8_t options[0]; + } __attribute__((packed)); + + struct NeighborSol + { + IP6::addr target; + uint8_t options[0]; + + IP6::addr get_target() + { return target; } + + } __attribute__((packed)); + + struct NeighborAdv + { + IP6::addr target; + uint8_t options[0]; + + void set_target(IP6::addr tar) + { target = tar; } + } __attribute__((packed)); + + public: + + using Span = gsl::span; + + static constexpr size_t header_size() + { return sizeof(Header); } + + Type type() const noexcept + { return header().type; } + + uint8_t code() const noexcept + { return header().code; } + + uint16_t checksum() const noexcept + { return header().checksum; } + + uint16_t id() const noexcept + { return header().identifier; } + + uint16_t sequence() const noexcept + { return header().sequence; } + + /** + * Where the payload of an ICMP packet starts, calculated from the start of the IP header + * The payload of an ICMP error message packet contains the original packet sent that caused an + * ICMP error to occur (the original IP header and 8 bytes of the original packet's data) + */ + int payload_index() + { return pckt_->ip_header_len() + header_size(); } + + Span payload() + { return {&(header().payload[0]), pckt_->data_end() - &(header().payload[0]) }; } + + /** Several ICMP messages require the payload to be the header and 64 bits of the + * data of the original datagram + */ + Span header_and_data() + { return {pckt_->layer_begin(), pckt_->ip_header_len() + 8}; } + + RouterSol& router_sol() + { return *reinterpret_cast(&(header().payload[0])); } + + RouterAdv& router_adv() + { return *reinterpret_cast(&(header().payload[0])); } + + RouterRedirect& router_redirect() + { return *reinterpret_cast(&(header().payload[0])); } + + NeighborSol& neighbor_sol() + { return *reinterpret_cast(&(header().payload[0])); } + + NeighborAdv& neighbor_adv() + { return *reinterpret_cast(&(header().payload[0])); } + + void set_type(Type t) noexcept + { header().type = t; } + + void set_code(uint8_t c) noexcept + { header().code = c; } + + void set_id(uint16_t s) noexcept + { header().identifier = s; } + + void set_sequence(uint16_t s) noexcept + { header().sequence = s; } + + void set_reserved(uint32_t s) noexcept + { header().reserved = s; } + + /** + * RFC 792 Parameter problem f.ex.: error (Pointer) is placed in the first byte after checksum + * (identifier and sequence is not used when pointer is used) + */ + void set_pointer(uint8_t error) + { header().identifier = error; } + + uint16_t compute_checksum() const noexcept + { + uint16_t datalen = ip().payload_length(); + pseudo_header phdr; + + // ICMP checksum is done with a pseudo header + // consisting of src addr, dst addr, message length (32bits) + // 3 zeroes (8bits each) and id of the next header + phdr.src = ip().ip_src(); + phdr.dst = ip().ip_dst(); + phdr.len = htonl(datalen); + phdr.zeros[0] = 0; + phdr.zeros[1] = 0; + phdr.zeros[2] = 0; + phdr.next = ip().next_header(); + assert(phdr.next == 58); // ICMPv6 + + /** + RFC 4443 + 2.3. Message Checksum Calculation + + The checksum is the 16-bit one's complement of the one's complement + sum of the entire ICMPv6 message, starting with the ICMPv6 message + type field, and prepended with a "pseudo-header" of IPv6 header + fields, as specified in [IPv6, Section 8.1]. The Next Header value + used in the pseudo-header is 58. (The inclusion of a pseudo-header + in the ICMPv6 checksum is a change from IPv4; see [IPv6] for the + rationale for this change.) + + For computing the checksum, the checksum field is first set to zero. + **/ + union + { + uint32_t whole; + uint16_t part[2]; + } sum; + sum.whole = 0; + + // compute sum of pseudo header + uint16_t* it = (uint16_t*) &phdr; + uint16_t* it_end = it + sizeof(pseudo_header) / 2; + + while (it < it_end) + sum.whole += *(it++); + + // compute sum of data + it = (uint16_t*) &header(); + it_end = it + datalen / 2; + + while (it < it_end) + sum.whole += *(it++); + + // odd-numbered case + if (datalen & 1) + sum.whole += *(uint8_t*) it; + + return ~(sum.part[0] + sum.part[1]); + } + + void set_checksum() { + ip().set_segment_length(); + header().checksum = 0; + header().checksum = compute_checksum();; + } + + void set_neighbor_adv_flag(uint32_t flag) + { header().rso_flags = htonl(flag << 28); } + + void set_payload(Span new_load) + { + pckt_->set_data_end(pckt_->ip_header_len() + header_size() + new_load.size()); + memcpy(payload().data(), new_load.data(), payload().size()); + } + + /** Get the underlying IP packet */ + IP6::IP_packet& ip() const + { return *pckt_.get(); } + + /** Construct from existing packet **/ + Packet(IP6::IP_packet_ptr pckt) + : pckt_{ std::move(pckt) } + { } + + /** Provision fresh packet from factory **/ + Packet(IP6::IP_packet_factory create) + : pckt_{create(Protocol::ICMPv6)} + { + pckt_->increment_data_end(sizeof(Header)); + } + + /** Release packet pointer **/ + IP6::IP_packet_ptr release() + { return std::move(pckt_); } + + private: + IP6::IP_packet_ptr pckt_; + }; +} +} diff --git a/api/net/ip6/packet_ip6.hpp b/api/net/ip6/packet_ip6.hpp index 10bea8134b..0131b535f3 100644 --- a/api/net/ip6/packet_ip6.hpp +++ b/api/net/ip6/packet_ip6.hpp @@ -38,15 +38,19 @@ namespace net ip6::Header& ip6_header() noexcept { return *reinterpret_cast(layer_begin()); } + uint32_t ver_tc_fl() const noexcept + { return ip6_header().ver_tc_fl; } + uint8_t ip6_version() const noexcept - { return ip6_header().version; } + { return (ver_tc_fl() & 0xF0) >> 4; } bool is_ipv6() const noexcept - { return (ip6_header().version) == 6; } + { return ip6_version() == 6; } /** Get traffic class */ uint8_t traffic_class() const noexcept - { return (ip6_header().traffic_class); } + { return ((ver_tc_fl() & 0xF0000) >> 12) + + (ver_tc_fl() & 0xF); } /** Get Differentiated Services Code Point (DSCP)*/ DSCP ip_dscp() const noexcept @@ -58,7 +62,7 @@ namespace net /** Get flow label */ uint32_t flow_label() const noexcept - { return (ip6_header().flow_label); } + { return (htonl(ver_tc_fl()) & 0x000FFFFF); } /** Get payload length */ uint16_t payload_length() const noexcept @@ -95,6 +99,11 @@ namespace net uint16_t ip_capacity() const noexcept { return capacity() - IP6_HEADER_LEN; } + /* This returns the IPv6 header and extension header len. + * Note: Extension header needs to be parsed to know this */ + uint16_t ip_header_len() const noexcept + { return IP6_HEADER_LEN + get_extension_header_len(); } + // IPv6 setters // /** Set IP version header field */ @@ -132,6 +141,12 @@ namespace net void set_ip_dst(const ip6::Addr& addr) noexcept { ip6_header().daddr = addr; } + void decrement_hop_limit() + { + Expects(hop_limit() != 0); + ip6_header().hop_limit--; + } + /** Last modifications before transmission */ void make_flight_ready() noexcept { assert(ip6_header().next_header); @@ -142,9 +157,9 @@ namespace net Expects(size() == 0); auto& hdr = ip6_header(); std::memset(&ip6_header(), 0, IP6_HEADER_LEN); - hdr.version = 6; + hdr.ver_tc_fl = 0x0060; hdr.next_header = static_cast(proto); - hdr.payload_length = 0x1400; // Big-endian 20 + hdr.payload_length = 0x0; increment_data_end(IP6_HEADER_LEN); } @@ -156,30 +171,42 @@ namespace net return {ip_data_ptr(), ip_data_length()}; } + void update_extension_header_len(uint8_t len) { + extension_header_len_ += len; + } + + uint16_t get_extension_header_len() const { + return extension_header_len_; + } + + /** + * Set IP6 payload length + */ + void set_segment_length() noexcept + { ip6_header().payload_length = htons(size() - IP6_HEADER_LEN); } + protected: /** Get pointer to IP data */ Byte* ip_data_ptr() noexcept __attribute__((assume_aligned(4))) { - return layer_begin() + IP6_HEADER_LEN; + return layer_begin() + IP6_HEADER_LEN + get_extension_header_len(); } const Byte* ip_data_ptr() const noexcept __attribute__((assume_aligned(4))) { - return layer_begin() + IP6_HEADER_LEN; + return layer_begin() + IP6_HEADER_LEN + get_extension_header_len(); } - private: - /** - * Set IP6 payload length - */ - void set_segment_length() noexcept - { ip6_header().payload_length = htons(size()) - IP6_HEADER_LEN; } + + private: const ip6::Header& ip6_header() const noexcept { return *reinterpret_cast(layer_begin()); } + uint16_t extension_header_len_; + }; //< class PacketIP6 } //< namespace net #endif diff --git a/etc/scripts/create_bridge.sh b/etc/scripts/create_bridge.sh index 251b7cd838..e8d30c5a19 100755 --- a/etc/scripts/create_bridge.sh +++ b/etc/scripts/create_bridge.sh @@ -9,6 +9,8 @@ then BRIDGE=bridge43 NETMASK=255.255.255.0 GATEWAY=10.0.0.1 + NETMASK6=64 + GATEWAY6=fe80::e823:fcff:fef4:83e7 elif [ $# -eq 3 ] then @@ -26,6 +28,7 @@ if [ -n "$INCLUDEOS_BRIDGE" ]; then fi echo " Creating bridge $BRIDGE, netmask $NETMASK, gateway $GATEWAY " +echo " ipv6 netmask $NETMASK6, gateway $GATEWAY6 " # HÃ¥reks cool hack: # - First two bytes is fixed to "c001" because it's cool @@ -84,6 +87,7 @@ else echo " Configuring network bridge (requires sudo):" sudo ifconfig $BRIDGE $GATEWAY netmask $NETMASK up || exit 1 + sudo ifconfig $BRIDGE inet6 add $GATEWAY6/$NETMASK6 if uname -s | grep Darwin > /dev/null 2>&1; then sudo ifconfig $BRIDGE ether $HWADDR || exit 1 else diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cb7789c85d..77d73dca6c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -47,7 +47,7 @@ set(OS_OBJECTS net/tcp/tcp.cpp net/tcp/connection.cpp net/tcp/connection_states.cpp net/tcp/write_queue.cpp net/tcp/rttm.cpp net/tcp/listener.cpp net/tcp/read_buffer.cpp net/ip4/icmp4.cpp net/ip4/udp.cpp net/ip4/udp_socket.cpp - net/ip6/ip6.cpp + net/ip6/ip6.cpp net/ip6/icmp6.cpp net/dns/dns.cpp net/dns/client.cpp net/dhcp/dh4client.cpp net/dhcp/dhcpd.cpp net/buffer_store.cpp net/inet.cpp net/super_stack.cpp net/configure.cpp net/conntrack.cpp net/vlan_manager.cpp diff --git a/src/net/inet.cpp b/src/net/inet.cpp index fa4683fcc0..05d7aeab3c 100644 --- a/src/net/inet.cpp +++ b/src/net/inet.cpp @@ -28,7 +28,7 @@ Inet::Inet(hw::Nic& nic) gateway_(IP4::ADDR_ANY), dns_server_(IP4::ADDR_ANY), nic_(nic), arp_(*this), ip4_(*this), ip6_(*this), - icmp_(*this), udp_(*this), tcp_(*this), dns_(*this), + icmp_(*this), icmp6_(*this), udp_(*this), tcp_(*this), dns_(*this), domain_name_{}, MTU_(nic.MTU()) { @@ -44,6 +44,7 @@ Inet::Inet(hw::Nic& nic) auto ip4_bottom(upstream_ip{ip4_, &IP4::receive}); auto ip6_bottom(upstream_ip{ip6_, &IP6::receive}); auto icmp4_bottom(upstream{icmp_, &ICMPv4::receive}); + auto icmp6_bottom(upstream{icmp6_, &ICMPv6::receive}); auto udp4_bottom(upstream{udp_, &UDP::receive}); auto tcp_bottom(upstream{tcp_, &TCP::receive}); @@ -63,6 +64,9 @@ Inet::Inet(hw::Nic& nic) // IP4 -> ICMP ip4_.set_icmp_handler(icmp4_bottom); + // IP6 -> ICMP6 + ip6_.set_icmp_handler(icmp6_bottom); + // IP4 -> UDP ip4_.set_udp_handler(udp4_bottom); @@ -80,6 +84,9 @@ Inet::Inet(hw::Nic& nic) // ICMP -> IP4 icmp_.set_network_out(ip4_top); + // ICMPv6 -> IP6 + icmp6_.set_network_out(ip6_top); + // UDP4 -> IP4 udp_.set_network_out(ip4_top); @@ -89,8 +96,9 @@ Inet::Inet(hw::Nic& nic) // IP4 -> Arp ip4_.set_linklayer_out(arp_top); - // ICMP6 -> IP6 - // icmp6->set_network_out(ip6_top); + // IP6 -> Link + ip6_.set_linklayer_out(link_top); + // UDP6 -> IP6 // udp6->set_network_out(ip6_top); // TCP6 -> IP6 @@ -100,7 +108,6 @@ Inet::Inet(hw::Nic& nic) // IP6 -> Link assert(link_top); arp_.set_linklayer_out(link_top); - //ip6_.set_linklayer_out(link_top); #ifndef INCLUDEOS_SINGLE_THREADED // move this nework stack automatically @@ -189,10 +196,7 @@ void Inet::negotiate_dhcp(double timeout, dhcp_timeout_func handler) { void Inet::network_config(IP4::addr addr, IP4::addr nmask, IP4::addr gateway, - IP4::addr dns, - IP6::addr addr6, - IP6::addr prefix6, - IP6::addr gateway6) + IP4::addr dns) { this->ip4_addr_ = addr; this->netmask_ = nmask; @@ -204,15 +208,24 @@ void Inet::network_config(IP4::addr addr, INFO2("Gateway: \t%s", gateway_.str().c_str()); INFO2("DNS Server: \t%s", dns_server_.str().c_str()); - if (addr6 != IP6::ADDR_ANY) { - this->ip6_addr_ = addr6; - this->ip6_prefix_ = prefix6; - this->ip6_gateway_ = gateway6; - INFO("Inet6", "Network configured (%s)", nic_.mac().to_string().c_str()); - INFO2("IP6: \t\t%s", ip6_addr_.str().c_str()); - INFO2("Prefix: \t%s", ip6_prefix_.str().c_str()); - INFO2("Gateway: \t%s", ip6_gateway_.str().c_str()); - } + for(auto& handler : configured_handlers_) + handler(*this); + + configured_handlers_.clear(); +} + +void Inet::network_config6(IP6::addr addr6, + uint8_t prefix6, + IP6::addr gateway6) +{ + + this->ip6_addr_ = addr6; + this->ip6_prefix_ = prefix6; + this->ip6_gateway_ = gateway6; + INFO("Inet6", "Network configured (%s)", nic_.mac().to_string().c_str()); + INFO2("IP6: \t\t%s", ip6_addr_.str().c_str()); + INFO2("Prefix: \t%d", ip6_prefix_); + INFO2("Gateway: \t%s", ip6_gateway_.str().c_str()); for(auto& handler : configured_handlers_) handler(*this); diff --git a/src/net/ip6/icmp6.cpp b/src/net/ip6/icmp6.cpp index 6e7c8170f8..14a3b6f4aa 100644 --- a/src/net/ip6/icmp6.cpp +++ b/src/net/ip6/icmp6.cpp @@ -15,9 +15,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -//#define DEBUG +//#define ICMP6_DEBUG 1 +#ifdef ICMP6_DEBUG +#define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define PRINT(fmt, ...) /* fmt */ +#endif #include -#include +#include #include #include @@ -26,304 +31,365 @@ namespace net { - // internal implementation of handler for ICMP type 128 (echo requests) - int echo_request(ICMPv6&, std::shared_ptr& pckt); - int neighbor_solicitation(ICMPv6& caller, std::shared_ptr& pckt); - - ICMPv6::ICMPv6(IP6::addr& local_ip) - : localIP(local_ip) - { - // install default handler for echo requests - listen(ECHO_REQUEST, echo_request); - // install default handler for neighbor solicitation requests - listen(ND_NEIGHB_SOL, neighbor_solicitation); + // ---------------------------- ICMP_view ---------------------------- + + std::string ICMP6_view::to_string() const { + if (type_ == ICMP_type::NO_REPLY) + return "No reply received"; + + return "Identifier: " + std::to_string(id_) + "\n" + + "Sequence number: " + std::to_string(seq_) + "\n" + + "Source: " + src_.to_string() + "\n" + + "Destination: " + dst_.to_string() + "\n" + + "Type: " + icmp6::get_type_string(type_) + "\n" + + "Code: " + icmp6::get_code_string(type_, code_) + "\n" + + "Checksum: " + std::to_string(checksum_) + "\n" + + "Data: " + payload_; } - // static function that returns a textual representation of - // the @code and @type of an ICMP message - std::string ICMPv6::code_string(uint8_t type, uint8_t code) + // ------------------------------ ICMPv6 ------------------------------ + + int ICMPv6::request_id_ = 0; + + ICMPv6::ICMPv6(Stack& inet) : + inet_{inet} + {} + + void ICMPv6::send_router_solicitation() { - switch (type) - { - /// error codes /// - case 1: - /// delivery problems /// - switch (code) - { - case 0: - return "No route to destination"; - case 1: - return "Communication with dest administratively prohibited"; - case 2: - return "Beyond scope of source address"; - case 3: - return "Address unreachable"; - case 4: - return "Port unreachable"; - case 5: - return "Source address failed ingress/egress policy"; - case 6: - return "Reject route to destination"; - case 7: - return "Error in source routing header"; - default: - return "ERROR Invalid ICMP type"; - } - case 2: - /// size problems /// - return "Packet too big"; - - case 3: - /// time problems /// - switch (code) - { - case 0: - return "Hop limit exceeded in traffic"; - case 1: - return "Fragment reassembly time exceeded"; - default: - return "ERROR Invalid ICMP code"; - } - case 4: - /// parameter problems /// - switch (code) - { - case 0: - return "Erroneous header field"; - case 1: - return "Unrecognized next header"; - case 2: - return "Unrecognized IPv6 option"; - default: - return "ERROR Invalid ICMP code"; - } - - /// echo feature /// - case ECHO_REQUEST: - return "Echo request"; - case ECHO_REPLY: - return "Echo reply"; - - /// multicast feature /// - case 130: - return "Multicast listener query"; - case 131: - return "Multicast listener report"; - case 132: - return "Multicast listener done"; - - /// neighbor discovery protocol /// - case ND_ROUTER_SOL: - return "NDP Router solicitation request"; - case ND_ROUTER_ADV: - return "NDP Router advertisement"; - case ND_NEIGHB_SOL: - return "NDP Neighbor solicitation request"; - case ND_NEIGHB_ADV: - return "NDP Neighbor advertisement"; - case ND_REDIRECT: - return "NDP Redirect message"; - - case 143: - return "Multicast Listener Discovery (MLDv2) reports (RFC 3810)"; - - default: - return "Unknown type: " + std::to_string((int) type); - } + icmp6::Packet req(inet_.ip6_packet_factory()); + req.ip().set_ip_src(inet_.ip6_addr()); + req.ip().set_ip_dst(ip6::Addr::node_all_nodes); + req.ip().set_ip_hop_limit(255); + req.set_type(ICMP_type::ND_ROUTER_SOLICATION); + req.set_code(0); + req.set_reserved(0); + // Set multicast addreqs + // IPv6mcast_02: 33:33:00:00:00:02 + + // Add checksum + req.set_checksum(); + + PRINT(" Router solicit size: %i payload size: %i, checksum: 0x%x\n", + req.ip().size(), req.payload().size(), req.compute_checksum()); + + network_layer_out_(req.release()); } - int ICMPv6::bottom(Packet_ptr pckt) + int ICMPv6::receive_neighbor_solicitation(icmp6::Packet& req) { - auto icmp = std::static_pointer_cast(pckt); - - type_t type = icmp->type(); - - if (listeners.find(type) != listeners.end()) - { - return listeners[type](*this, icmp); - } - else - { - debug(">>> IPv6 -> ICMPv6 bottom (no handler installed)\n"); - debug("ICMPv6 type %d: %s\n", - (int) icmp->type(), code_string(icmp->type(), icmp->code()).c_str()); - - /* - // show correct checksum - intptr_t chksum = icmp->checksum(); - debug("ICMPv6 checksum: %p \n",(void*) chksum); - - // show our recalculated checksum - icmp->header().checksum_ = 0; - chksum = checksum(icmp); - debug("ICMPv6 our estimate: %p \n", (void*) chksum ); - */ + bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; + IP6::addr target = req.neighbor_sol().get_target(); + + PRINT("ICMPv6 NDP Neighbor solicitation request\n"); + PRINT(">> target: %s\n", target.str().c_str()); + + if (target.is_multicast()) { + PRINT("ND: neighbor solictation target address is multicast\n"); return -1; - } + } + + icmp6::Packet res(inet_.ip6_packet_factory()); + + // drop if the packet is too small + if (res.ip().capacity() < IP6_HEADER_LEN + + (int) res.header_size() + req.payload().size()) + { + printf("WARNING: Network MTU too small for ICMP response, dropping\n"); + return -1; + } + + // Populate response IP header + res.ip().set_ip_src(inet_.ip6_addr()); + if (any_src) { + res.ip().set_ip_dst(ip6::Addr::node_all_nodes); + } else { + res.ip().set_ip_dst(req.ip().ip_src()); + } + + // Populate response ICMP header + res.set_type(ICMP_type::ND_NEIGHBOR_ADV); + res.set_code(0); + res.set_neighbor_adv_flag(NEIGH_ADV_SOL | NEIGH_ADV_OVERRIDE); + PRINT(" Transmitting Neighbor adv to %s\n", + res.ip().ip_dst().str().c_str()); + + // Payload + res.set_payload(req.payload()); + + // Add checksum + res.set_checksum(); + + PRINT(" Neighbor Adv Response size: %i payload size: %i, checksum: 0x%x\n", + res.ip().size(), res.payload().size(), res.compute_checksum()); + + network_layer_out_(res.release()); + return 0; } - int ICMPv6::transmit(std::shared_ptr& pckt) - { - // NOTE: *** OBJECT CREATED ON STACK *** --> - auto original = std::static_pointer_cast(pckt); - // NOTE: *** OBJECT CREATED ON STACK *** <-- - return ip6_out(original); + + void ICMPv6::receive(Packet_ptr pckt) { + if (not is_full_header((size_t) pckt->size())) // Drop if not a full header + return; + + auto pckt_ip6 = static_unique_ptr_cast(std::move(pckt)); + auto req = icmp6::Packet(std::move(pckt_ip6)); + PRINT(" receive from %s\n", req.ip().ip_src().to_string().c_str()); + + switch(req.type()) { + case (ICMP_type::ECHO): + PRINT(" PING from %s\n", req.ip().ip_src().to_string().c_str()); + ping_reply(req); + break; + case (ICMP_type::ECHO_REPLY): + PRINT(" PING Reply from %s\n", req.ip().ip_src().str().c_str()); + execute_ping_callback(req); + break; + case (ICMP_type::DEST_UNREACHABLE): + PRINT(" Destination unreachable from %s\n", req.ip().ip_src().str().c_str()); + break; + case (ICMP_type::PACKET_TOO_BIG): + PRINT(" ICMP Too Big message from %s\n", req.ip().ip_src().str().c_str()); + handle_too_big(req); + return; + case (ICMP_type::TIME_EXCEEDED): + case (ICMP_type::PARAMETER_PROBLEM): + PRINT(" ICMP error message from %s\n", req.ip().ip_src().str().c_str()); + forward_to_transport_layer(req); + break; + case (ICMP_type::MULTICAST_LISTENER_QUERY): + case (ICMP_type::MULTICAST_LISTENER_REPORT): + case (ICMP_type::MULTICAST_LISTENER_DONE): + PRINT(" ICMP multicast message from %s\n", req.ip().ip_src().str().c_str()); + break; + case (ICMP_type::ND_ROUTER_SOLICATION): + PRINT(" ICMP Router solictation message from %s\n", req.ip().ip_src().str().c_str()); + break; + case (ICMP_type::ND_ROUTER_ADV): + PRINT(" ICMP Router advertisement message from %s\n", req.ip().ip_src().str().c_str()); + break; + case (ICMP_type::ND_NEIGHBOR_SOL): + PRINT(" ICMP Neigbor solictation message from %s\n", req.ip().ip_src().str().c_str()); + receive_neighbor_solicitation(req); + break; + case (ICMP_type::ND_NEIGHBOR_ADV): + PRINT(" ICMP Neigbor advertisement message from %s\n", req.ip().ip_src().str().c_str()); + break; + case (ICMP_type::ND_REDIRECT): + PRINT(" ICMP Neigbor redirect message from %s\n", req.ip().ip_src().str().c_str()); + break; + case (ICMP_type::ROUTER_RENUMBERING): + PRINT(" ICMP Router re-numbering message from %s\n", req.ip().ip_src().str().c_str()); + break; + case (ICMP_type::INFORMATION_QUERY): + case (ICMP_type::INFORMATION_RESPONSE): + break; + default: // ICMP_type::NO_REPLY + return; + } } - uint16_t ICMPv6::checksum(std::shared_ptr& pckt) - { - IP6::header& hdr = pckt->ip6_header(); - - uint16_t datalen = hdr.size(); - pseudo_header phdr; - - // ICMP checksum is done with a pseudo header - // consisting of src addr, dst addr, message length (32bits) - // 3 zeroes (8bits each) and id of the next header - phdr.src = hdr.src; - phdr.dst = hdr.dst; - phdr.len = htonl(datalen); - phdr.zeros[0] = 0; - phdr.zeros[1] = 0; - phdr.zeros[2] = 0; - phdr.next = hdr.next(); - //assert(hdr.next() == 58); // ICMPv6 - - /** - RFC 4443 - 2.3. Message Checksum Calculation - - The checksum is the 16-bit one's complement of the one's complement - sum of the entire ICMPv6 message, starting with the ICMPv6 message - type field, and prepended with a "pseudo-header" of IPv6 header - fields, as specified in [IPv6, Section 8.1]. The Next Header value - used in the pseudo-header is 58. (The inclusion of a pseudo-header - in the ICMPv6 checksum is a change from IPv4; see [IPv6] for the - rationale for this change.) - - For computing the checksum, the checksum field is first set to zero. - **/ - union - { - uint32_t whole; - uint16_t part[2]; - } sum; - sum.whole = 0; + void ICMPv6::forward_to_transport_layer(icmp6::Packet& req) { + ICMP6_error err{req.type(), req.code()}; - // compute sum of pseudo header - uint16_t* it = (uint16_t*) &phdr; - uint16_t* it_end = it + sizeof(pseudo_header) / 2; + // The icmp6::Packet's payload contains the original packet sent that resulted + // in an error + int payload_idx = req.payload_index(); + auto packet_ptr = req.release(); + packet_ptr->increment_layer_begin(payload_idx); - while (it < it_end) - sum.whole += *(it++); + // inet forwards to transport layer (UDP or TCP) + inet_.error_report(err, std::move(packet_ptr)); + } - // compute sum of data - it = (uint16_t*) pckt->payload(); - it_end = it + datalen / 2; + void ICMPv6::handle_too_big(icmp6::Packet& req) { + // In this type of ICMP packet, the Next-Hop MTU is placed at the same location as + // the sequence number in an ECHO message f.ex. + ICMP6_error err{req.type(), req.code(), req.sequence()}; - while (it < it_end) - sum.whole += *(it++); + // The icmp6::Packet's payload contains the original packet sent that resulted + // in the Fragmentation Needed + int payload_idx = req.payload_index(); + auto packet_ptr = req.release(); + packet_ptr->increment_layer_begin(payload_idx); - // odd-numbered case - if (datalen & 1) - sum.whole += *(uint8_t*) it; + // Inet updates the corresponding Path MTU value in IP and notifies the transport/packetization layer + inet_.error_report(err, std::move(packet_ptr)); + } - return ~(sum.part[0] + sum.part[1]); + void ICMPv6::destination_unreachable(Packet_ptr pckt, icmp6::code::Dest_unreachable code) { + if (not is_full_header((size_t) pckt->size())) // Drop if not a full header + return; + + auto pckt_ip6 = static_unique_ptr_cast(std::move(pckt)); + +#if 0 + // Only sending destination unreachable message if the destination IP address is not + // broadcast or multicast + if (pckt_ip6->ip_dst() != inet_.broadcast_addr() and pckt_ip6->ip_dst() != IP6::ADDR_BCAST and + not pckt_ip6->ip_dst().is_multicast()) { + auto pckt_icmp6 = icmp6::Packet(std::move(pckt_ip6)); + send_response(pckt_icmp6, ICMP_type::DEST_UNREACHABLE, (ICMP_code) code); + } +#endif } - // internal implementation of handler for ICMP type 128 (echo requests) - int echo_request(ICMPv6& caller, std::shared_ptr& pckt) - { - ICMPv6::echo_header* icmp = (ICMPv6::echo_header*) pckt->payload(); - debug("*** Custom handler for ICMP ECHO REQ type=%d 0x%x\n", icmp->type, htons(icmp->checksum)); - - // set the hoplimit manually to the very standard 64 hops - pckt->set_hoplimit(64); - - // set to ICMP Echo Reply (129) - icmp->type = ICMPv6::ECHO_REPLY; - - if (pckt->dst().is_multicast()) - { - // We won't be changing source address for multicast ping - debug("Was multicast ping6: no change for source and dest\n"); - } - else - { - printf("Normal ping6: source is us\n"); - printf("src is %s\n", pckt->src().str().c_str()); - printf("dst is %s\n", pckt->dst().str().c_str()); - - printf("multicast is %s\n", IP6::addr::link_all_nodes.str().c_str()); - // normal ping: send packet to source, from us - pckt->set_dst(pckt->src()); - pckt->set_src(caller.local_ip()); - } - // calculate and set checksum - // NOTE: do this after changing packet contents! - icmp->checksum = 0; - icmp->checksum = ICMPv6::checksum(pckt); - - // send packet downstream - return caller.transmit(pckt); + void ICMPv6::time_exceeded(Packet_ptr pckt, icmp6::code::Time_exceeded code) { + if (not is_full_header((size_t) pckt->size())) // Drop if not a full header + return; + + auto pckt_ip6 = static_unique_ptr_cast(std::move(pckt)); + auto pckt_icmp6 = icmp6::Packet(std::move(pckt_ip6)); + send_response(pckt_icmp6, ICMP_type::TIME_EXCEEDED, (ICMP_code) code); } - int neighbor_solicitation(ICMPv6& caller, std::shared_ptr& pckt) - { - (void) caller; - NDP::neighbor_sol* sol = (NDP::neighbor_sol*) pckt->payload(); - printf("ICMPv6 NDP Neighbor solicitation request\n"); - printf(">> target: %s\n", sol->target.str().c_str()); - printf(">>\n"); - printf(">> source: %s\n", pckt->src().str().c_str()); - printf(">> dest: %s\n", pckt->dst().str().c_str()); + void ICMPv6::parameter_problem(Packet_ptr pckt, uint8_t error_pointer) { + if (not is_full_header((size_t) pckt->size())) // Drop if not a full header + return; - // perhaps we should answer - (void) caller; + auto pckt_ip6 = static_unique_ptr_cast(std::move(pckt)); + auto pckt_icmp6 = icmp6::Packet(std::move(pckt_ip6)); + send_response(pckt_icmp6, ICMP_type::PARAMETER_PROBLEM, 0, error_pointer); + } + + void ICMPv6::ping(IP6::addr ip) + { send_request(ip, ICMP_type::ECHO, 0); } - return -1; + void ICMPv6::ping(IP6::addr ip, icmp_func callback, int sec_wait) + { send_request(ip, ICMP_type::ECHO, 0, callback, sec_wait); } + + void ICMPv6::ping(const std::string& hostname) { +#if 0 + inet_.resolve(hostname, [this] (IP6::addr a, Error err) { + if (!err and a != IP6::ADDR_ANY) + ping(a); + }); +#endif } - void ICMPv6::discover() - { - // ether-broadcast an IPv6 packet to all routers - // IPv6mcast_02: 33:33:00:00:00:02 - auto pckt = IP6::create( - IP6::PROTO_ICMPv6, - Ethernet::IPv6mcast_02, - IP6::addr::link_unspecified); - - // RFC4861 4.1. Router Solicitation Message Format - pckt->set_hoplimit(255); - - NDP::router_sol* ndp = (NDP::router_sol*) pckt->payload(); - // set to Router Solicitation Request - ndp->type = ICMPv6::ND_ROUTER_SOL; - ndp->code = 0; - ndp->checksum = 0; - ndp->reserved = 0; - - auto icmp = std::static_pointer_cast (pckt); - - // source and destination addresses - icmp->set_src(this->local_ip()); //IP6::addr::link_unspecified); - icmp->set_dst(IP6::addr::link_all_routers); - - // ICMP header length field - icmp->set_length(sizeof(NDP::router_sol)); - - // calculate and set checksum - // NOTE: do this after changing packet contents! - ndp->checksum = ICMPv6::checksum(icmp); - - this->transmit(icmp); - - /// DHCPv6 test /// - // ether-broadcast an IPv6 packet to all routers - //pckt = IP6::create( - // IP6::PROTO_UDP, - // Ethernet::IPv6mcast_02, - // IP6::addr::link_unspecified); + void ICMPv6::ping(const std::string& hostname, icmp_func callback, int sec_wait) { +#if 0 + inet_.resolve(hostname, Inet::resolve_func::make_packed([this, callback, sec_wait] (IP6::addr a, Error err) { + if (!err and a != IP6::ADDR_ANY) + ping(a, callback, sec_wait); + })); +#endif + } + + void ICMPv6::send_request(IP6::addr dest_ip, ICMP_type type, ICMP_code code, + icmp_func callback, int sec_wait, uint16_t sequence) { + + // Check if inet is configured with ipv6 + if (!inet_.is_configured_v6()) { + PRINT(" inet is not configured to send ipv6 packets\n"); + return; + } + // Provision new IP6-packet + icmp6::Packet req(inet_.ip6_packet_factory()); + + // Populate request IP header + req.ip().set_ip_src(inet_.ip6_addr()); + req.ip().set_ip_dst(dest_ip); + + uint16_t temp_id = request_id_; + // Populate request ICMP header + req.set_type(type); + req.set_code(code); + req.set_id(request_id_++); + req.set_sequence(sequence); + + if (callback) { + ping_callbacks_.emplace(std::piecewise_construct, + std::forward_as_tuple(std::make_pair(temp_id, sequence)), + std::forward_as_tuple(ICMP_callback{*this, std::make_pair(temp_id, sequence), callback, sec_wait})); + } + + PRINT(" Transmitting request to %s\n", dest_ip.to_string().c_str()); + + // Payload + // Default: includeos_payload_ + req.set_payload(icmp6::Packet::Span(includeos_payload_, 68)); + + // Add checksum + req.set_checksum(); + + PRINT(" Request size: %i payload size: %i, checksum: 0x%x\n", + req.ip().size(), req.payload().size(), req.compute_checksum()); + + network_layer_out_(req.release()); + } + + void ICMPv6::send_response(icmp6::Packet& req, ICMP_type type, ICMP_code code, uint8_t error_pointer) { + + // Check if inet is configured with ipv6 + if (!inet_.is_configured_v6()) { + PRINT(" inet is not configured to send ipv6 response\n"); + return; + } + + // Provision new IP6-packet + icmp6::Packet res(inet_.ip6_packet_factory()); + + // Populate response IP header + res.ip().set_ip_src(inet_.ip6_addr()); + res.ip().set_ip_dst(req.ip().ip_src()); + + // Populate response ICMP header + res.set_type(type); + res.set_code(code); + + PRINT(" Transmitting answer to %s\n", res.ip().ip_dst().str().c_str()); + + // Payload + // Default: Header and 66 bits (8 bytes) of original payload + res.set_payload(req.header_and_data()); + + // Add checksum + res.set_checksum(); + + if (error_pointer != std::numeric_limits::max()) + res.set_pointer(error_pointer); + + PRINT(" Response size: %i payload size: %i, checksum: 0x%x\n", + res.ip().size(), res.payload().size(), res.compute_checksum()); + + network_layer_out_(res.release()); } + void ICMPv6::ping_reply(icmp6::Packet& req) { + // Provision new IP6-packet + icmp6::Packet res(inet_.ip6_packet_factory()); + + // drop if the packet is too small + if (res.ip().capacity() < IP6_HEADER_LEN + + (int) res.header_size() + req.payload().size()) + { + printf("WARNING: Network MTU too small for ICMP response, dropping\n"); + return; + } + // Populate response IP header + res.ip().set_ip_src(inet_.ip6_addr()); + res.ip().set_ip_dst(req.ip().ip_src()); + + // Populate response ICMP header + res.set_type(ICMP_type::ECHO_REPLY); + res.set_code(0); + // Incl. id and sequence number + res.set_id(req.id()); + res.set_sequence(req.sequence()); + + PRINT(" Transmitting answer to %s\n", + res.ip().ip_dst().str().c_str()); + + // Payload + res.set_payload(req.payload()); + + // Add checksum + res.set_checksum(); + + PRINT(" Response size: %i payload size: %i, checksum: 0x%x\n", + res.ip().size(), res.payload().size(), res.compute_checksum()); + + network_layer_out_(res.release()); + } } diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index 7db82a824c..c4ede5671c 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -89,26 +89,28 @@ namespace net or local_ip() == ADDR_ANY; } - void IP6::ipv6_ext_header_receive(Packet_ptr pckt) + Protocol IP6::ipv6_ext_header_receive(net::PacketIP6& packet) { - auto packet = static_unique_ptr_cast(std::move(pckt)); - uint8_t* reader = packet->buf(); - auto header = packet->ip6_header(); - reader += sizeof(header); + auto reader = packet.layer_begin() + IP6_HEADER_LEN; + auto next_proto = packet.next_protocol(); ip6::ExtensionHeader& ext = *(ip6::ExtensionHeader*)reader; - auto next_proto = packet->next_protocol(); + uint8_t ext_len; while (next_proto != Protocol::IPv6_NONXT) { if (next_proto == Protocol::HOPOPT) { + PRINT("HOP extension header\n"); } else if (next_proto == Protocol::OPTSV6) { } else { - PRINT("Unknown IPv6 extension header\n"); - break; + PRINT("Done parsing extension header, next proto: %d\n", next_proto); + return next_proto; } ext = *(ip6::ExtensionHeader*)reader; - reader += ext.size(); + ext_len = ext.size(); + reader += ext_len; + packet.update_extension_header_len(ext_len); next_proto = ext.next(); } + return next_proto; } void IP6::receive(Packet_ptr pckt, const bool link_bcast) @@ -116,11 +118,13 @@ namespace net // Cast to IP6 Packet auto packet = static_unique_ptr_cast(std::move(pckt)); - PRINT(" Source IP: %s Dest.IP: %s Type: %d LinkBcast: %d ", + PRINT(" Source IP: %s, Dest.IP: %s, Next header: %d," + "Payload len: %u, Hop limit: %d, version: %d, tc: %u, fl: %u\n", packet->ip_src().str().c_str(), packet->ip_dst().str().c_str(), (int) packet->next_header(), - link_bcast); + packet->payload_length(), packet->hop_limit(), + packet->ip6_version(), packet->traffic_class(), packet->flow_label()); switch (packet->next_protocol()) { case Protocol::HOPOPT: @@ -198,29 +202,33 @@ namespace net packet = res.release(); #endif + auto next_proto = packet->next_protocol(); // Pass packet to it's respective protocol controller - switch (packet->next_protocol()) { - case Protocol::HOPOPT: - case Protocol::OPTSV6: - ipv6_ext_header_receive(std::move(pckt)); - break; + if (packet->next_protocol() == Protocol::HOPOPT || + packet->next_protocol() == Protocol::OPTSV6) { + next_proto = ipv6_ext_header_receive(*packet); + } + + switch (next_proto) { case Protocol::IPv6_NONXT: /* Nothing after the icmp header */ break; case Protocol::ICMPv6: + PRINT("ICMPv6: %d\n", packet->size()); icmp_handler_(std::move(packet)); break; case Protocol::UDP: - udp_handler_(std::move(packet)); + //udp_handler_(std::move(packet)); break; case Protocol::TCP: - tcp_handler_(std::move(packet)); + //tcp_handler_(std::move(packet)); break; default: // Send ICMP error of type Destination Unreachable and code PROTOCOL // @note: If dest. is broadcast or multicast it should be dropped by now //stack_.icmp().destination_unreachable(std::move(packet), icmp6::code::Dest_unreachable::PROTOCOL); + PRINT("Unknown next proto. Dropping packet\n"); drop(std::move(packet), Direction::Upstream, Drop_reason::Unknown_proto); break; } @@ -310,13 +318,13 @@ namespace net // Stat increment packets transmitted packets_tx_++; -#if 0 extern MAC::Addr linux_tap_device; - MAC::Addr dest_mac = linux_tap_device; + MAC::Addr dest_mac("c0:01:0a:00:00:01"); - PRINT(" Transmitting packet, layer begin: buf + %li\n", packet->layer_begin() - packet->buf()); + PRINT(" Transmitting packet on mac address: %s," + " layer begin: buf + %li\n", dest_mac.to_string().c_str(), + packet->layer_begin() - packet->buf()); linklayer_out_(std::move(packet), dest_mac, Ethertype::IP6); -#endif } const ip6::Addr IP6::local_ip() const { diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index 7dde0dcbfc..ae4928fd7d 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -15,4 +15,4 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8264380337..ab303dc2f2 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -101,6 +101,8 @@ set(TEST_SOURCES ${TEST}/net/unit/ip4.cpp ${TEST}/net/unit/ip4_packet_test.cpp ${TEST}/net/unit/ip6.cpp + ${TEST}/net/unit/ip6_addr.cpp + ${TEST}/net/unit/ip6_packet_test.cpp ${TEST}/net/unit/nat_test.cpp ${TEST}/net/unit/napt_test.cpp ${TEST}/net/unit/packets.cpp @@ -189,6 +191,7 @@ set(OS_SOURCES ${SRC}/net/ip4/reassembly.cpp ${SRC}/net/ip4/udp.cpp ${SRC}/net/ip4/udp_socket.cpp + ${SRC}/net/ip6/icmp6.cpp ${SRC}/net/ip6/ip6.cpp ${SRC}/net/dhcp/dh4client.cpp ${SRC}/net/nat/nat.cpp diff --git a/test/lest_util/packet_factory.hpp b/test/lest_util/packet_factory.hpp index 2f911da4a1..4743994ac6 100644 --- a/test/lest_util/packet_factory.hpp +++ b/test/lest_util/packet_factory.hpp @@ -58,6 +58,25 @@ static std::unique_ptr create_ip4_packet_init(ip4::Addr src, ip4 return ip4; } +#include +static std::unique_ptr create_ip6_packet() noexcept +{ + auto pkt = create_packet(); + pkt->increment_layer_begin(sizeof(net::ethernet::Header)); + // IP6 Packet + auto ip6 = net::static_unique_ptr_cast (std::move(pkt)); + return ip6; +} + +static std::unique_ptr create_ip6_packet_init(ip6::Addr src, ip6::Addr dst) noexcept +{ + auto ip6 = create_ip6_packet(); + ip6->init(); + ip6->set_ip_src(src); + ip6->set_ip_dst(dst); + return ip6; +} + #include static std::unique_ptr create_tcp_packet() noexcept { diff --git a/test/net/integration/ipv6/CMakeLists.txt b/test/net/integration/icmp6/CMakeLists.txt similarity index 71% rename from test/net/integration/ipv6/CMakeLists.txt rename to test/net/integration/icmp6/CMakeLists.txt index da27f7519d..c036709709 100644 --- a/test/net/integration/ipv6/CMakeLists.txt +++ b/test/net/integration/icmp6/CMakeLists.txt @@ -7,13 +7,13 @@ endif() include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) -project(test_ipv6) +project(test_icmp) # Human-readable name of your service -set(SERVICE_NAME "IncludeOS IPv6 test") +set(SERVICE_NAME "IncludeOS ICMP test") # Name of your service binary -set(BINARY "test_ipv6") +set(BINARY "test_icmp") # Maximum memory can be hard-coded into the binary set(MAX_MEM 128) @@ -24,10 +24,15 @@ set(SOURCES ) # DRIVERS / PLUGINS: - -set(DRIVERS - virtionet +if ("$ENV{PLATFORM}" STREQUAL "x86_solo5") + set(DRIVERS + solo5net # Virtio networking + ) +else() + set(DRIVERS + virtionet ) +endif() # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/net/integration/ipv6/service.cpp b/test/net/integration/icmp6/service.cpp similarity index 50% rename from test/net/integration/ipv6/service.cpp rename to test/net/integration/icmp6/service.cpp index 2c550f072d..509e067abe 100644 --- a/test/net/integration/ipv6/service.cpp +++ b/test/net/integration/icmp6/service.cpp @@ -23,11 +23,47 @@ using namespace net; void Service::start(const std::string&) { auto& inet = Inet::stack<0>(); - inet.network_config({10,0,0,42}, {255,255,255,0}, {10,0,0,1}, {0, 0, 0, 0}, + inet.network_config({ 10, 0, 0, 45 }, // IP + { 255, 255, 0, 0 }, // Netmask + { 10, 0, 0, 1 }, // Gateway + { 8, 8, 8, 8 } // DNS + ); + + inet.network_config6( { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd }, // IP6 - { 0, 0, 0, 64 }, // Prefix6 + 64, // Prefix6 { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x83e7 } // Gateway6 ); + printf("Service IPv4 address: %s, IPv6 address: %s\n", inet.ip_addr().str().c_str(), inet.ip6_addr().str().c_str()); + + // ping gateway + inet.icmp6().ping(inet.gateway6(), [](ICMP6_view pckt) { + if (pckt) + printf("Received packet from gateway\n%s\n", pckt.to_string().c_str()); + else + printf("No reply received from gateway\n"); + }); + +#if 0 + const int wait = 10; + + // No reply-pings + // Waiting 30 seconds for reply + inet.icmp().ping(IP4::addr{10,0,0,42}, [](ICMP_view pckt) { + if (pckt) + printf("Received packet from 10.0.0.42\n%s\n", pckt.to_string().c_str()); + else + printf("No reply received from 10.0.0.42\n"); + }, wait); + + // Waiting 30 seconds for reply + inet.icmp().ping(IP4::addr{10,0,0,43}, [](ICMP_view pckt) { + if (pckt) + printf("Received packet from 10.0.0.43\n%s\n", pckt.to_string().c_str()); + else + printf("No reply received from 10.0.0.43\n"); + }, wait); +#endif } diff --git a/test/net/integration/ipv6/vm.json b/test/net/integration/icmp6/vm.json similarity index 78% rename from test/net/integration/ipv6/vm.json rename to test/net/integration/icmp6/vm.json index 5dea28b28d..cae9a2a028 100644 --- a/test/net/integration/ipv6/vm.json +++ b/test/net/integration/icmp6/vm.json @@ -1,5 +1,5 @@ { - "image" : "test_ipv6.img", + "image" : "test_icmp.img", "net" : [{"device" : "virtio", "mac" : "c0:01:0a:00:00:2a"}], "mem" : 128, "intrusive" : "True" diff --git a/test/net/unit/ip6.cpp b/test/net/unit/ip6.cpp index 4364d3e184..c34e65cc2a 100644 --- a/test/net/unit/ip6.cpp +++ b/test/net/unit/ip6.cpp @@ -37,7 +37,7 @@ CASE("IP6 packet setters/getters") EXPECT(ip_pckt1->ip_dscp() == DSCP::CS0); EXPECT(ip_pckt1->ip_ecn() == ECN::NOT_ECT); EXPECT(ip_pckt1->flow_label() == 0); - EXPECT(ip_pckt1->payload_length() == 20); + EXPECT(ip_pckt1->payload_length() == 0); EXPECT(ip_pckt1->next_protocol() == Protocol::ICMPv4); EXPECT(ip_pckt1->hop_limit() == 0); EXPECT(ip_pckt1->ip_src() == IP6::ADDR_ANY); diff --git a/test/net/unit/ip6_addr.cpp b/test/net/unit/ip6_addr.cpp new file mode 100644 index 0000000000..c729723a5a --- /dev/null +++ b/test/net/unit/ip6_addr.cpp @@ -0,0 +1,74 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +using namespace net::ip6; + +CASE("Creating a empty IP6 address using the different constructors yields the same result") +{ + const std::string empty_addr_str {"0.0.0.0"}; + + const Addr addr1; + const Addr addr2 {0, 0, 0, 0}; + const Addr addr3 {0,0,0,0}; + + EXPECT( addr1 == addr2 ); + EXPECT( addr2 == addr3 ); +} + +CASE("IP6 addresses can be compared to each other") +{ + Addr addr1 { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd }; + Addr addr2 { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x84aa }; + Addr addr3 { }; + + EXPECT_NOT( addr1 == addr2 ); + EXPECT_NOT( addr1 == addr3 ); + EXPECT( addr2 != addr3 ); + + const Addr temp { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd }; + EXPECT( addr1 == temp ); + + const Addr empty; + EXPECT_NOT( addr1 == empty ); + + EXPECT( addr1 > addr2 ); + EXPECT( addr1 > addr3 ); + EXPECT( addr2 > addr3 ); + EXPECT( addr3 < addr1 ); + EXPECT( addr2 <= addr1 ); + + uint8_t netmask = 64; + const Addr not_terrorist { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd }; + + Addr result = not_terrorist & netmask; + const Addr expected_result { 0xfe80, 0, 0, 0, 0, 0, 0, 0 }; + EXPECT( result == expected_result ); +} + +CASE("Determine if an address is loopback") +{ + Addr l1 { 0,0,0,1 }; + Addr l2 { 127,10,0,42 }; + Addr l3 { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd }; + + EXPECT(l1.is_loopback()); + EXPECT(not l2.is_loopback()); + EXPECT(not l3.is_loopback()); +} diff --git a/test/net/unit/ip6_packet_test.cpp b/test/net/unit/ip6_packet_test.cpp new file mode 100644 index 0000000000..68441eaa3f --- /dev/null +++ b/test/net/unit/ip6_packet_test.cpp @@ -0,0 +1,66 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +using namespace net; + +CASE("IP6 Packet HOPLIMIT") +{ + const ip6::Addr src{0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd}; + const ip6::Addr dst{ 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x83e7 }; + auto ip6 = create_ip6_packet_init(src, dst); + + const uint8_t DEFAULT_HOPLIMIT = 128; + + ip6->set_ip_hop_limit(DEFAULT_HOPLIMIT); + + EXPECT(ip6->hop_limit() == DEFAULT_HOPLIMIT); + + for(uint8_t i = DEFAULT_HOPLIMIT; i > 0; i--) { + ip6->decrement_hop_limit(); + EXPECT(ip6->hop_limit() == i-1); + } +} + +CASE("IP6 Packet HOPLIMIT - multiple packets") +{ + std::vector addrs{ + {0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd}, + {0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x83e7}, + {0,0,0,1}, + {0xfe80, 0, 0, 0, 0x0202, 0xb3ff, 0xff1e, 0x8329}, + }; + + for(int i = 0; i < addrs.size()-1; i++) + { + auto ip6 = create_ip6_packet_init(addrs[i], addrs[i+1]); + + const uint8_t DEFAULT_HOPLIMIT = 64; + + ip6->set_ip_hop_limit(DEFAULT_HOPLIMIT); + + EXPECT(ip6->hop_limit() == DEFAULT_HOPLIMIT); + + for(uint8_t i = DEFAULT_HOPLIMIT; i > 0; i--) + ip6->decrement_hop_limit(); + + EXPECT(ip6->hop_limit() == 0); + } +} From 494af10a16b64d1855cdc5e044b193efa50951b4 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Thu, 10 May 2018 11:31:37 +0530 Subject: [PATCH 243/723] liveupdate: Fix unit test failure. --- examples/LiveUpdate/service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/service.cpp index 18147dc285..44a1b8475b 100644 --- a/examples/LiveUpdate/service.cpp +++ b/examples/LiveUpdate/service.cpp @@ -53,7 +53,7 @@ static void save_state(liu::Storage& store, const liu::buffer_t*) static void resume_state(liu::Restore& thing) { printf("Resume state called\n"); - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); while (not thing.is_marker()) { From 7e64d12ee8e3eb30f070c3ef146d445b93735857 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Thu, 10 May 2018 01:49:48 -0700 Subject: [PATCH 244/723] ip6: Revert ip6 address changes. This is simpler --- api/net/ip6/addr.hpp | 60 +++++++++++++++++++++---------------------- src/net/ip6/icmp6.cpp | 5 ++-- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index 41d8501428..a556941b9b 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -46,16 +46,16 @@ struct Addr { Addr(uint16_t a1, uint16_t a2, uint16_t b1, uint16_t b2, uint16_t c1, uint16_t c2, uint16_t d1, uint16_t d2) { - i16[7] = a1; i16[6] = a2; - i16[5] = b1; i16[4] = b2; - i16[3] = c1; i16[2] = c2; - i16[1] = d1; i16[0] = d2; + i16[0] = htons(a1); i16[1] = htons(a2); + i16[2] = htons(b1); i16[3] = htons(b2); + i16[4] = htons(c1); i16[5] = htons(c2); + i16[6] = htons(d1); i16[7] = htons(d2); } Addr(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { - i32[0] = d; i32[1] = c; - i32[2] = b; i32[3] = a; + i32[0] = htonl(a); i32[1] = htonl(b); + i32[2] = htonl(c); i32[3] = htonl(d); } Addr(const Addr& a) @@ -70,9 +70,9 @@ struct Addr { char ipv6_addr[32]; snprintf(ipv6_addr, sizeof(ipv6_addr), "%0x:%0x:%0x:%0x:%0x:%0x:%0x:%0x", - i16[7], i16[6], i16[5], - i16[4], i16[3], i16[2], - i16[1], i16[0]); + ntohs(i16[0]), ntohs(i16[1]), ntohs(i16[2]), + ntohs(i16[3]), ntohs(i16[4]), ntohs(i16[5]), + ntohs(i16[6]), ntohs(i16[7])); return ipv6_addr; } @@ -94,8 +94,8 @@ struct Addr { // RFC 4291 2.4.6: // Link-Local Addresses are designed to be used for Addressing on a - // single link for purposes such as automatic Address configuration, // neighbor discovery, or when no routers are present. + // single link for purposes such as automatic Address configuration, static const Addr link_all_nodes; // RFC 4921 static const Addr link_all_routers; // RFC 4921 static const Addr link_mDNSv6; // RFC 6762 @@ -124,8 +124,8 @@ struct Addr { * **/ bool is_loopback() const noexcept - { return i32[0] == 1 && i32[1] == 0 && - i32[2] == 0 && i32[3] == 0; } + { return i32[0] == 0 && i32[1] == 0 && + i32[2] == 0 && ntohl(i32[3]) == 1; } /** * Assignment operator @@ -156,10 +156,10 @@ struct Addr { */ bool operator>(const Addr other) const noexcept { - if (i32[3] > other.i32[3]) return true; - if (i32[2] > other.i32[2]) return true; - if (i32[1] > other.i32[1]) return true; if (i32[0] > other.i32[0]) return true; + if (i32[1] > other.i32[1]) return true; + if (i32[2] > other.i32[2]) return true; + if (i32[3] > other.i32[3]) return true; return false; } @@ -188,21 +188,21 @@ struct Addr { * IPv6 addresses */ Addr operator&(const Addr other) const noexcept - { return Addr{i32[3] & other.i32[3], - i32[2] & other.i32[2], + { return Addr{i32[0] & other.i32[0], i32[1] & other.i32[1], - i32[0] & other.i32[0] }; } + i32[2] & other.i32[2], + i32[3] & other.i32[3] }; } Addr operator&(uint8_t prefix) const noexcept { - int i = 0; + int i = 3; uint8_t mask; uint32_t addr[32]; - addr[0] = i32[0]; - addr[1] = i32[1]; - addr[2] = i32[2]; - addr[3] = i32[3]; + addr[0] = htonl(i32[0]); + addr[1] = htonl(i32[1]); + addr[2] = htonl(i32[2]); + addr[3] = htonl(i32[3]); if (prefix > 128) { prefix = 128; @@ -210,25 +210,25 @@ struct Addr { mask = 128 - prefix; while (mask >= 32) { - addr[i++] = 0; + addr[i--] = 0; mask -= 32; } if (mask != 0) { addr[i] &= (0xFFFFFFFF << mask); } - return Addr { addr[3], addr[2], - addr[1], addr[0] }; + return Addr { addr[0], addr[1], + addr[2], addr[3] }; } Addr operator|(const Addr other) const noexcept - { return Addr{i32[3] | other.i32[3], - i32[2] | other.i32[2], + { return Addr{i32[0] | other.i32[0], i32[1] | other.i32[1], - i32[0] | other.i32[0] }; } + i32[2] | other.i32[2], + i32[3] | other.i32[3] }; } Addr operator~() const noexcept - { return Addr{~i32[3], ~i32[2], ~i32[1], ~i32[0]}; } + { return Addr{~i32[0], ~i32[1], ~i32[2], ~i32[3]}; } union { diff --git a/src/net/ip6/icmp6.cpp b/src/net/ip6/icmp6.cpp index 14a3b6f4aa..05e3108915 100644 --- a/src/net/ip6/icmp6.cpp +++ b/src/net/ip6/icmp6.cpp @@ -95,7 +95,7 @@ namespace net if (res.ip().capacity() < IP6_HEADER_LEN + (int) res.header_size() + req.payload().size()) { - printf("WARNING: Network MTU too small for ICMP response, dropping\n"); + PRINT("WARNING: Network MTU too small for ICMP response, dropping\n"); return -1; } @@ -106,6 +106,7 @@ namespace net } else { res.ip().set_ip_dst(req.ip().ip_src()); } + res.ip().set_ip_hop_limit(255); // Populate response ICMP header res.set_type(ICMP_type::ND_NEIGHBOR_ADV); @@ -363,7 +364,7 @@ namespace net if (res.ip().capacity() < IP6_HEADER_LEN + (int) res.header_size() + req.payload().size()) { - printf("WARNING: Network MTU too small for ICMP response, dropping\n"); + PRINT("WARNING: Network MTU too small for ICMP response, dropping\n"); return; } From 68756dcd7af0d752746f889f27ba445328f72ee1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 11 May 2018 09:14:41 +0200 Subject: [PATCH 245/723] websocket: HTTPS upgrade can return nullptr --- api/net/ws/connector.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/net/ws/connector.hpp b/api/net/ws/connector.hpp index 6d46ada760..82ee2731a9 100644 --- a/api/net/ws/connector.hpp +++ b/api/net/ws/connector.hpp @@ -91,11 +91,9 @@ class WS_server_connector : public WS_connector { } } auto ws = WebSocket::upgrade(*req, *writer); - assert(ws->get_cpuid() == SMP::cpu_id()); - - if(ws == nullptr) { - } // not ok + if (ws == nullptr) return; + assert(ws->get_cpuid() == SMP::cpu_id()); on_connect_(std::move(ws)); } From f63f9eb3674e7802e407560b2066c561ea49b41e Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 11 May 2018 09:34:27 +0200 Subject: [PATCH 246/723] examples: Benchmark mode for HTTPS server, add Botan support --- examples/TLS_server/config.json | 2 +- examples/TLS_server/service.cpp | 32 ++++++++++++++++++++++++++++---- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/examples/TLS_server/config.json b/examples/TLS_server/config.json index 44b02e3db3..26564e1325 100644 --- a/examples/TLS_server/config.json +++ b/examples/TLS_server/config.json @@ -2,7 +2,7 @@ "net" : [ { "iface": 0, - "config": "dhcp-with-fallback", + "config": "static", "address": "10.0.0.42", "netmask": "255.255.255.0", "gateway": "10.0.0.1" diff --git a/examples/TLS_server/service.cpp b/examples/TLS_server/service.cpp index 9249a29ca3..31d76e2020 100644 --- a/examples/TLS_server/service.cpp +++ b/examples/TLS_server/service.cpp @@ -15,28 +15,53 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +#include #include #include #include #include static http::Server* server = nullptr; extern http::Response_ptr handle_request(const http::Request&); +static const bool ENABLE_TLS = true; +static const bool USE_BOTAN_TLS = true; +#define BENCHMARK void Service::start() { + fs::memdisk().init_fs( + [] (auto err, auto&) { + assert(!err); + }); // Get the first IP stack // It should have configuration from config.json auto& inet = net::Super_stack::get(0); +#ifndef BENCHMARK // Print some useful netstats every 30 secs Timers::periodic(5s, 30s, [&inet] (uint32_t) { printf(" TCP STATUS:\n%s\n", inet.tcp().status().c_str()); }); +#endif + + if (USE_BOTAN_TLS) { + auto& filesys = fs::memdisk().fs(); + // load CA certificate + auto ca_cert = filesys.stat("/test.der"); + // load CA private key + auto ca_key = filesys.stat("/test.key"); + // load server private key + auto srv_key = filesys.stat("/server.key"); - server = new http::OpenSSL_server( - "/test.pem", "/test.key", inet.tcp()); + server = new http::Botan_server( + "blabla", ca_key, ca_cert, srv_key, inet.tcp()); + printf("Using Botan for HTTPS transport\n"); + } + else { + server = new http::OpenSSL_server( + "/test.pem", "/test.key", inet.tcp()); + printf("Using OpenSSL for HTTPS transport\n"); + } server->on_request( [] (auto request, auto response_writer) { @@ -46,5 +71,4 @@ void Service::start() // listen on default HTTPS port server->listen(443); - printf("*** TLS service started ***\n"); } From 3225d73779f0b2cbbc49d27980ee90876efc0104 Mon Sep 17 00:00:00 2001 From: Taiyeba Date: Fri, 11 May 2018 10:26:12 +0200 Subject: [PATCH 247/723] Changing IP addresses for dhcpd integration test --- test/net/integration/README.md | 1 + test/net/integration/dhcpd/service.cpp | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/net/integration/README.md b/test/net/integration/README.md index 7af532edc7..4dda2e2a55 100644 --- a/test/net/integration/README.md +++ b/test/net/integration/README.md @@ -7,3 +7,4 @@ The following IP addresses are used by the services in the test folder. Make sur - tcp: 10.0.0.44 - udp: 10.0.0.45 - websocket: 10.0.0.54 +- dhcpd: 10.0.0.1, 10.0.0.10, 10.0.0.20 diff --git a/test/net/integration/dhcpd/service.cpp b/test/net/integration/dhcpd/service.cpp index 758425b295..59db449902 100644 --- a/test/net/integration/dhcpd/service.cpp +++ b/test/net/integration/dhcpd/service.cpp @@ -30,13 +30,13 @@ void Service::start(const std::string&) // Server auto& inet = Inet::ifconfig<0>( - { 10,0,100,1 }, // IP + { 10,0,0,1 }, // IP { 255,255,255,0 }, // Netmask { 10,0,0,1 }, // Gateway { 8,8,8,8 }); // DNS - IP4::addr pool_start{10,0,100,10}; - IP4::addr pool_end{10,0,100,20}; + IP4::addr pool_start{10,0,0,10}; + IP4::addr pool_end{10,0,0,20}; server = std::make_unique(inet.udp(), pool_start, pool_end); // Client 1 From b7901a75a7ebeacd3ced6ec532dffd5c4a10265b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 11 May 2018 11:20:48 +0200 Subject: [PATCH 248/723] util: Allow Fixed_alloc copy-const for compile with GCC7, init storage on first alloc --- api/util/fixed_list_alloc.hpp | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/api/util/fixed_list_alloc.hpp b/api/util/fixed_list_alloc.hpp index 0954c6c8a5..6be4eb3b22 100644 --- a/api/util/fixed_list_alloc.hpp +++ b/api/util/fixed_list_alloc.hpp @@ -45,18 +45,18 @@ class Fixed_list_alloc { static_assert((sizeof(T) * size) % alignment == 0, "Total size (sizeof(T) * N) needs to be a multiple of alignment Align"); - + using propagate_on_container_move_assignment = std::true_type; private: std::unique_ptr store_; public: Fixed_list_alloc() - : store_{std::make_unique()} + : store_{nullptr} {} template Fixed_list_alloc(const U&) - : store_{std::make_unique()} + : store_{nullptr} { // ain't much to do here since storage is fixed to T, // so we just create a new storage for this type @@ -65,10 +65,26 @@ class Fixed_list_alloc { Fixed_list_alloc(Fixed_list_alloc&&) noexcept = default; Fixed_list_alloc& operator=(Fixed_list_alloc&&) noexcept = default; + // I'd really like to disable copy construction, since it makes no sense here. + // >=Clang5 and >=GCC8 allows this ('rebind' resp. 'move constuctor' being used), + // but GCC7 does not (unecessary utilize copy constructor). + Fixed_list_alloc(const Fixed_list_alloc&) + : store_{nullptr} + { + // we can't really copy due to fixed_storage, + // just create new storage and hope no assumptions + // are made that the allocators are the same + } + /* No copy */ - Fixed_list_alloc(const Fixed_list_alloc&) = delete; + //Fixed_list_alloc(const Fixed_list_alloc&) = delete; Fixed_list_alloc& operator=(const Fixed_list_alloc&) = delete; + static Fixed_list_alloc select_on_container_copy_construction(const Fixed_list_alloc&) + { + return {}; + } + template using other = Fixed_list_alloc; @@ -78,6 +94,8 @@ class Fixed_list_alloc { T* allocate(std::size_t n) { + if(UNLIKELY(store_ == nullptr)) + store_.reset(new storage_type); return reinterpret_cast(store_->template allocate(n*sizeof(T))); } From d53be89be6672c14840511c1aa68878600acdf87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 11 May 2018 12:12:48 +0200 Subject: [PATCH 249/723] test: Temporary skip path_to_regex_no_options test --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 1bc416d651..2f78530f5e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -125,7 +125,7 @@ set(TEST_SOURCES ${TEST}/util/unit/isotime.cpp ${TEST}/util/unit/logger_test.cpp ${TEST}/util/unit/membitmap.cpp - ${TEST}/util/unit/path_to_regex_no_options.cpp + #${TEST}/util/unit/path_to_regex_no_options.cpp ${TEST}/util/unit/path_to_regex_parse.cpp ${TEST}/util/unit/path_to_regex_options.cpp ${TEST}/util/unit/percent_encoding_test.cpp From 6448b5c5a585ef19186415fdeb13abdb448bfb11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 11 May 2018 15:16:51 +0200 Subject: [PATCH 250/723] test: Include malloc if not Mac in lstack test --- test/util/unit/lstack.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/util/unit/lstack.cpp b/test/util/unit/lstack.cpp index 8740091be6..7b44da950b 100644 --- a/test/util/unit/lstack.cpp +++ b/test/util/unit/lstack.cpp @@ -16,7 +16,11 @@ #include #include +#ifdef __MACH__ extern void* memalign(size_t alignment, size_t size); +#else +#include +#endif using Lstack = util::alloc::Lstack<>; using Chunk = Lstack::Chunk; From 3466ef8d99c16a65b0969d2e63e1800d780f99f5 Mon Sep 17 00:00:00 2001 From: Taiyeba Date: Fri, 11 May 2018 15:17:44 +0200 Subject: [PATCH 251/723] Update LiveUpdate service to fit with new Super_stack interface --- examples/LiveUpdate/service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/service.cpp index 18147dc285..44a1b8475b 100644 --- a/examples/LiveUpdate/service.cpp +++ b/examples/LiveUpdate/service.cpp @@ -53,7 +53,7 @@ static void save_state(liu::Storage& store, const liu::buffer_t*) static void resume_state(liu::Restore& thing) { printf("Resume state called\n"); - auto& inet = net::Super_stack::get(0); + auto& inet = net::Super_stack::get(0); while (not thing.is_marker()) { From a8c838e712534568111f7ea44c913efab3e19c25 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Sun, 13 May 2018 18:37:34 +0530 Subject: [PATCH 252/723] ip6: Use std::array instead of regular arrays --- api/net/ip6/addr.hpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index a556941b9b..38dbfe3dc4 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -67,7 +67,7 @@ struct Addr { // returns this IPv6 Address as a string std::string str() const { - char ipv6_addr[32]; + char ipv6_addr[40]; snprintf(ipv6_addr, sizeof(ipv6_addr), "%0x:%0x:%0x:%0x:%0x:%0x:%0x:%0x", ntohs(i16[0]), ntohs(i16[1]), ntohs(i16[2]), @@ -117,7 +117,7 @@ struct Addr { |11111111|flgs|scop| group ID | +--------+----+----+---------------------------------------------+ **/ - return i8[0] == 0xFF; + return ((ntohs(i16[0]) & 0xFF00) == 0xFF00); } /** @@ -210,15 +210,15 @@ struct Addr { mask = 128 - prefix; while (mask >= 32) { - addr[i--] = 0; + addr[i--] = 0; mask -= 32; } if (mask != 0) { - addr[i] &= (0xFFFFFFFF << mask); + addr[i] &= (0xFFFFFFFF << mask); } - return Addr { addr[0], addr[1], - addr[2], addr[3] }; + return Addr { addr[0], addr[1], + addr[2], addr[3] }; } Addr operator|(const Addr other) const noexcept @@ -232,9 +232,9 @@ struct Addr { union { - uint32_t i32[4]; - uint16_t i16[8]; - uint8_t i8[16]; + std::array i64; + std::array i32; + std::array i16; }; } __attribute__((packed)); //< struct Addr } //< namespace ip6 From d0599bfd6beeb4184131b3a1aa209e58ee60fdf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 14 May 2018 10:12:25 +0200 Subject: [PATCH 253/723] test: Don't build unreliable cpu_test --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2f78530f5e..8e22b32b51 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -72,7 +72,7 @@ set(TEST_SOURCES ${TEST}/fs/unit/memdisk_test.cpp ${TEST}/fs/unit/path_test.cpp ${TEST}/fs/unit/vfs_test.cpp - ${TEST}/hw/unit/cpu_test.cpp + #${TEST}/hw/unit/cpu_test.cpp ${TEST}/hw/unit/mac_addr_test.cpp ${TEST}/hw/unit/nic_test.cpp ${TEST}/kernel/unit/events_test.cpp From 3bceebde15454983263df28c9f8a6a63d8d3caea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 14 May 2018 10:19:43 +0200 Subject: [PATCH 254/723] test: removed unecessary check in bitops --- test/util/unit/bitops.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/util/unit/bitops.cpp b/test/util/unit/bitops.cpp index dd50811cf1..885eaada68 100644 --- a/test/util/unit/bitops.cpp +++ b/test/util/unit/bitops.cpp @@ -104,8 +104,6 @@ CASE ("util::bitops: Using bitmask ops an enum and an integral") CASE ("util::bitops: using various bit operations") { - - EXPECT(__builtin_clzl(0) == 64); EXPECT(__builtin_clzl(0x1000) == 51); EXPECT(bits::keeplast(0x10110) == 0x10000); From 0bdb12ddae5b79ecde4b09ed6a22b1709abf1c2b Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 14 May 2018 10:23:29 +0200 Subject: [PATCH 255/723] net: Work on TLS streams --- api/net/botan/tls_server.hpp | 2 +- api/net/openssl/tls_stream.hpp | 24 ++++++++++---------- examples/TLS_server/service.cpp | 39 +++++++++++++++++++++++---------- src/net/http/server.cpp | 4 ---- 4 files changed, 40 insertions(+), 29 deletions(-) diff --git a/api/net/botan/tls_server.hpp b/api/net/botan/tls_server.hpp index 615b6e0ded..0255b5814b 100644 --- a/api/net/botan/tls_server.hpp +++ b/api/net/botan/tls_server.hpp @@ -77,7 +77,7 @@ class Server : public Botan::TLS::Callbacks, public net::Stream void close() override { m_transport->close(); CloseCallback cb = std::move(m_on_close); - reset_callbacks(); + this->reset_callbacks(); if (cb) cb(); } void abort() override { diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index 7805780728..ddf033ed5b 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -4,7 +4,7 @@ #include #include -//#define VERBOSE_OPENSSL +#define VERBOSE_OPENSSL #ifdef VERBOSE_OPENSSL #define TLS_PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else @@ -120,7 +120,7 @@ namespace openssl SSL_set_bio(this->m_ssl, this->m_bio_rd, this->m_bio_wr); // always-on callbacks m_transport->on_read(8192, {this, &TLS_stream::tls_read}); - m_transport->on_close({this, &TLS_stream::close_callback_once}); + m_transport->on_close({this, &TLS_stream::close}); // start TLS handshake process if (outgoing == true) @@ -143,7 +143,7 @@ namespace openssl inline void TLS_stream::write(buffer_t buffer) { if (UNLIKELY(this->is_connected() == false)) { - printf("TLS_stream::write() called on closed stream\n"); + TLS_PRINT("TLS_stream::write() called on closed stream\n"); return; } @@ -219,12 +219,13 @@ namespace openssl if (n > 0) { auto buf = net::tcp::construct_buffer(temp, temp + n); if (m_on_read) m_on_read(std::move(buf)); + } } while (n > 0); // this goes here? if (UNLIKELY(this->is_closing() || this->is_closed())) { - //this->close_callback_once(); - return; + TLS_PRINT("TLS_stream::SSL_read closed during read\n"); + return; } auto status = this->status(n); @@ -295,18 +296,14 @@ namespace openssl { ERR_clear_error(); m_transport->close(); + CloseCallback func = std::move(m_on_close); + this->reset_callbacks(); + if (func) func(); } inline void TLS_stream::abort() { m_transport->abort(); - this->reset_callbacks(); - } - inline void TLS_stream::close_callback_once() - { - auto func = std::move(m_on_close); - // free captured resources - this->reset_callbacks(); - if (func) func(); + this->close(); } inline void TLS_stream::reset_callbacks() { @@ -314,6 +311,7 @@ namespace openssl this->m_on_connect = nullptr; this->m_on_read = nullptr; this->m_on_write = nullptr; + this->m_transport->reset_callbacks(); // why this? } inline bool TLS_stream::handshake_completed() const noexcept diff --git a/examples/TLS_server/service.cpp b/examples/TLS_server/service.cpp index 31d76e2020..9b41192374 100644 --- a/examples/TLS_server/service.cpp +++ b/examples/TLS_server/service.cpp @@ -20,11 +20,12 @@ #include #include #include +#define BENCHMARK_MODE +static const bool ENABLE_TLS = true; +static const bool USE_BOTAN_TLS = false; + static http::Server* server = nullptr; extern http::Response_ptr handle_request(const http::Request&); -static const bool ENABLE_TLS = true; -static const bool USE_BOTAN_TLS = true; -#define BENCHMARK void Service::start() { @@ -32,12 +33,11 @@ void Service::start() [] (auto err, auto&) { assert(!err); }); - // Get the first IP stack - // It should have configuration from config.json + // Auto-configured from config.json auto& inet = net::Super_stack::get(0); -#ifndef BENCHMARK - // Print some useful netstats every 30 secs +#ifndef BENCHMARK_MODE + // Print some useful TCP stats every 30 secs Timers::periodic(5s, 30s, [&inet] (uint32_t) { printf(" TCP STATUS:\n%s\n", inet.tcp().status().c_str()); @@ -46,11 +46,8 @@ void Service::start() if (USE_BOTAN_TLS) { auto& filesys = fs::memdisk().fs(); - // load CA certificate - auto ca_cert = filesys.stat("/test.der"); - // load CA private key + auto ca_cert = filesys.stat("/test.pem"); auto ca_key = filesys.stat("/test.key"); - // load server private key auto srv_key = filesys.stat("/server.key"); server = new http::Botan_server( @@ -72,3 +69,23 @@ void Service::start() // listen on default HTTPS port server->listen(443); } + +#ifdef BENCHMARK_MODE +#include +static void print_heap_info() +{ + const std::string heapinfo = HeapDiag::to_string(); + printf("%s\n", heapinfo.c_str()); + StackSampler::print(10); +} + +void Service::ready() +{ + using namespace std::chrono; + Timers::periodic(1s, [] (int) { + print_heap_info(); + }); + + StackSampler::begin(); +} +#endif diff --git a/src/net/http/server.cpp b/src/net/http/server.cpp index de4cee210f..21b98c74fa 100644 --- a/src/net/http/server.cpp +++ b/src/net/http/server.cpp @@ -92,10 +92,6 @@ namespace http { { const auto idx = conn.idx(); connections_[idx] = nullptr; - if (free_idx_.capacity() < connections_.size()) - { - free_idx_.reserve(connections_.size()); - } free_idx_.push_back(idx); } From c1f77922967dc25640c744dc519c1f2e6b669f03 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Mon, 14 May 2018 11:45:19 +0200 Subject: [PATCH 256/723] gitignore: Added installation folder ignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4c030ae675..8d4ba4046b 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ test/**/*.log CMakeFiles* CMakeCache* cmake_install.cmake + +# Name of installation folder +IncludeOS_install From f66bbac88c1e4e3f6b4c83916ecae6e90bb33ab7 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Mon, 14 May 2018 11:48:19 +0200 Subject: [PATCH 257/723] Clang: Added check/prediction for clang version to separate script. This lets all sub-scripts that need to use a specific compiler just run this script in-line. This was needed for the chainloader. --- etc/build_chainloader.sh | 3 ++ etc/install_from_bundle.sh | 29 +---------- etc/use_clang_version.sh | 98 ++++++++++++++++++++++++++++++++++++++ install.sh | 6 ++- 4 files changed, 107 insertions(+), 29 deletions(-) create mode 100755 etc/use_clang_version.sh diff --git a/etc/build_chainloader.sh b/etc/build_chainloader.sh index 4b604d6f4c..aa0ea18f63 100755 --- a/etc/build_chainloader.sh +++ b/etc/build_chainloader.sh @@ -23,6 +23,9 @@ fi $INCLUDEOS_SRC/etc/install_from_bundle.sh echo -e "\n\n>>> Building chainloader" +# Set compiler version +source $INCLUDEOS_SRC/etc/use_clang_version.sh +echo -e "\n\n>>> Best guess for compatible compilers: $CXX / $CC" mkdir -p $CHAINLOAD_LOC/build pushd $CHAINLOAD_LOC/build diff --git a/etc/install_from_bundle.sh b/etc/install_from_bundle.sh index 3a1e941700..4d6f05fbce 100755 --- a/etc/install_from_bundle.sh +++ b/etc/install_from_bundle.sh @@ -19,33 +19,8 @@ set -e : ${num_jobs:="-j 4"} : ${ARCH:=x86_64} -# Try to find suitable compiler -cc_list="clang-7.0 clang-6.0 clang-5.0 clang-4.0 clang-3.9 clang" -cxx_list="clang++-7.0 clang++-6.0 clang++-5.0 clang++-4.0 clang++-3.9 clang++" - -compiler="" -guess_compiler() { - for compiler in $* - do - if command -v $compiler; then - break - fi - done -} - - -if [ -z "$CC" ] -then - guess_compiler "$cc_list" - export CC=$compiler -fi - -if [ -z "$CXX" ] -then - guess_compiler "$cxx_list" - export CXX=$compiler -fi - +# Set compiler version +source $INCLUDEOS_SRC/etc/use_clang_version.sh echo -e "\n\n>>> Best guess for compatible compilers: $CXX / $CC" # Build IncludeOS diff --git a/etc/use_clang_version.sh b/etc/use_clang_version.sh new file mode 100755 index 0000000000..a04bf3b400 --- /dev/null +++ b/etc/use_clang_version.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +# env variables +: ${INCLUDEOS_SRC:=$HOME/IncludeOS} +: ${INCLUDEOS_PREFIX:=/usr/local} +: ${CC:=""} +: ${CXX:=""} + +########################################################### +# Compiler information: +############################################################ +# This information can be changed to add/remove suport for a compiler version +CLANG_VERSION_MIN_REQUIRED="5.0" +cc_list="clang-7.0 clang-6.0 clang-5.0 clang" +cxx_list="clang++-7.0 clang++-6.0 clang++-5.0 clang++" + +############################################################ +# Support functions: +############################################################ +# newer_than_minimum takes a specific compiler vesion and checks if it is compatible +# returns error if it is not compatible +newer_than_minimum() { + local check=${1?You must specify which compiler to check} + local CLANG_VERSION=${check: -3} + if [[ $CLANG_VERSION < $CLANG_VERSION_MIN_REQUIRED ]]; then + return 1 + fi + return 0 +} + +check_installed() { + local clang=${1?Clang version needs to be set} + if [[ $(command -v $clang) ]]; then + return 0 + fi + return 1 +} + +# guess_compiler takes a list of compilers and returns the first one that is installed +check_if_in_list() { + for compiler in $*; do + if check_installed $compiler; then + compiler_to_use=$compiler + return 0 + fi + done + return 1 +} + +# containes checks if an item $1 is in the list supplied $2 +contains() { + for item in $2; do + if [ "$1" == "$item" ]; then + return 0 + fi + done + return 1 +} + +############################################################ +# Main: +############################################################ +# If CC and CXX is set then a check is performed to see if it compatible +# + +if [ "$CC" != "" ]; then + if ! newer_than_minimum "$CC"; then + echo Your clang version: $CC is older than minimum required: $CLANG_VERSION_MIN_REQUIRED + fi + if ! contains "$CC" "$cc_list"; then + echo Your clang version: $CC is not in the approved list: "$cc_list" + fi +else + check_if_in_list "$cc_list" + if [ $? -eq 1 ]; then + echo "None of the compatible CC's were installed: $cc_list" + else + echo Set CC=$compiler_to_use + export CC=$compiler_to_use + fi +fi + +if [ "$CXX" != "" ]; then + if ! newer_than_minimum "$CXX"; then + echo Your clang version: $CXX is older than minimum required: $CLANG_VERSION_MIN_REQUIRED + fi + if ! contains "$CXX" "$cxx_list"; then + echo Your clang version: $CXX is not in the approved list: "$cxx_list" + fi +else + check_if_in_list "$cxx_list" + if [ $? -eq 1 ]; then + echo "None of the compatible CC's were installed: $cxx_list" + else + echo Set CXX=$compiler_to_use + export CXX=$compiler_to_use + fi +fi diff --git a/install.sh b/install.sh index 2d66be7ee5..c4af520f68 100755 --- a/install.sh +++ b/install.sh @@ -254,10 +254,12 @@ fi # INSTALL FINISHED: ############################################################ +# Set compiler version +source $INCLUDEOS_SRC/etc/use_clang_version.sh printf "\n\n>>> IncludeOS installation Done!\n" printf " %s\n" "To use IncludeOS set env variables for cmake to know your compiler, e.g.:"\ - ' export CC="clang-5.0"'\ - ' export CXX="clang++-5.0"'\ + ' export CC="'$CC'"'\ + ' export CXX="'$CXX'"'\ ""\ "Test your installation with ./test.sh" From 6eda4dc2250ed7e4bc796a73febddd64a5b143b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 14 May 2018 13:41:01 +0200 Subject: [PATCH 258/723] tcp: Store end seq in buffer in case of reset during user callback --- src/net/tcp/read_request.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/net/tcp/read_request.cpp b/src/net/tcp/read_request.cpp index c6449dfa52..e613578957 100644 --- a/src/net/tcp/read_request.cpp +++ b/src/net/tcp/read_request.cpp @@ -55,6 +55,7 @@ namespace tcp { while(buf->is_ready() and buf == buffers.front().get()) { const auto rem = buf->capacity() - buf->size(); + const auto end_seq = buf->end_seq(); // store end_seq if reseted in callback if (callback) callback(buf->buffer()); // this is the only one, so we can reuse it @@ -63,7 +64,7 @@ namespace tcp { // Trick to make SACK work. If a lot of data was cleared up // it means the local sequence number is much farther behind // the real one - seq = buf->end_seq() - rem; + seq = end_seq - rem; buf->reset(seq); //printf("size=1, reset rem=%u start=%u end=%u\n", From 0ce494854bd51daaff7caee3e7f2c59c2504e34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 14 May 2018 13:41:45 +0200 Subject: [PATCH 259/723] ws: Swap to network endian for WS close reason --- src/net/ws/websocket.cpp | 4 ++-- test/net/integration/websocket/test.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/net/ws/websocket.cpp b/src/net/ws/websocket.cpp index bf287f7e94..0957967a12 100644 --- a/src/net/ws/websocket.cpp +++ b/src/net/ws/websocket.cpp @@ -300,10 +300,10 @@ void WebSocket::finalize_message() } return; case op_code::CLOSE: - // they are angry with us :( + // there is a message behind the reason, hmm.. if (hdr.data_length() >= 2) { // provide reason to user - uint16_t reason = *(uint16_t*) message->data(); + uint16_t reason = htons(*(uint16_t*) message->data()); this->close(reason); } else { diff --git a/test/net/integration/websocket/test.py b/test/net/integration/websocket/test.py index fd7c0d5897..599e2f3adc 100755 --- a/test/net/integration/websocket/test.py +++ b/test/net/integration/websocket/test.py @@ -21,7 +21,7 @@ def closed(self, code, reason=None): print " Closed down", code, reason def received_message(self, m): - print " received message" + #print " received message" self.count += 1 if self.count >= 1000: self.close(reason='Bye bye') From 1a6336a2f21bdb61f2bd90b8766a21563efb0f5d Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Mon, 14 May 2018 14:49:48 +0200 Subject: [PATCH 260/723] paging: add mem::protect for any-sized ranges, clean up pml4 alloc/dealloc --- api/arch/x86/paging.hpp | 256 +++++++++++++++++---- api/kernel/memory.hpp | 147 ++++++++---- src/arch/x86_64/paging.cpp | 114 ++++++--- src/platform/x86_pc/os.cpp | 2 +- test/kernel/integration/paging/service.cpp | 8 +- test/kernel/unit/memory.cpp | 231 +++++++++++++++++-- test/kernel/unit/x86_paging.cpp | 14 +- 7 files changed, 610 insertions(+), 162 deletions(-) diff --git a/api/arch/x86/paging.hpp b/api/arch/x86/paging.hpp index a3147fe5d9..947616c065 100644 --- a/api/arch/x86/paging.hpp +++ b/api/arch/x86/paging.hpp @@ -89,15 +89,6 @@ enum class Flags : uintptr_t { using Map = os::mem::Mapping; -/** 4k aligned allocation of page table / page dir **/ -template -T* allocate_pdir(Args... args) -{ - void* ptr = aligned_alloc(4_KiB, sizeof(T)); - Expects(ptr != nullptr); - return new (ptr) T(args...); -} - /** x86_64 specific types for page directories **/ struct x86_64 { using Flags = x86::paging::Flags; @@ -107,6 +98,8 @@ struct x86_64 { static constexpr uintptr_t max_memory = 512_GiB * 512; }; +static constexpr uintptr_t any_addr = std::numeric_limits::max(); + } } @@ -205,15 +198,16 @@ inline std::ostream& operator<<(std::ostream& out, const Summary& sum){ * (x86) Backend for os::mem::map / os::mem::protect **/ template + typename Arc = x86_64, + typename Arc::Flags Afl = Arc::Flags::all> class Page_table { public: - using Pflag = typename Platform::Flags; + using Arch = Arc; + using Pflag = typename Arch::Flags; using Subdir = Sub; static constexpr uintptr_t page_size = Psz; - static constexpr uintptr_t min_pagesize = Platform::min_pagesize; - static constexpr uintptr_t range_size = page_size * Platform::table_size; + static constexpr uintptr_t min_pagesize = Arch::min_pagesize; + static constexpr uintptr_t range_size = page_size * Arch::table_size; static constexpr Pflag allowed_flags = Afl; constexpr size_t size() { @@ -221,16 +215,46 @@ class Page_table { } Page_table() = default; - Page_table(uintptr_t start, Pflag flags = Pflag::none) - : linear_addr_start_{start} + Page_table(uintptr_t vstart) + : linear_addr_start_{vstart} { - // Expect 4k page alignment for all instances + //static_assert(std::is_pod::value, "Page table must be POD"); + static_assert(std::is_pod::value, + "Page table must have standard layout"); + static_assert(offsetof(Page_table, tbl_) == 0); + static_assert(alignof(Page_table) == Arch::min_pagesize); Expects((((uintptr_t)this) & ((uintptr_t)(4_KiB - 1))) == 0); + } + Page_table(uintptr_t lin, Pflag flags) + : Page_table(lin) + { // Default to identity-mapping - map_all(start, flags); + id_map(flags); } - static bool is_page_aligned(uintptr_t addr) noexcept + Page_table(uintptr_t lin, uintptr_t phys, Pflag flags) + : Page_table(lin) + { + map_all(phys, flags); + } + + Page_table(const Page_table&) = delete; + Page_table(const Page_table&&) = delete; + Page_table& operator=(const Page_table&) = delete; + Page_table& operator=(Page_table&&) = delete; + + ~Page_table() { + if constexpr (not std::is_void_v) { + for (auto& ent : tbl_) { + if (is_page_dir(ent)) { + Subdir* ptr = page_dir(&ent); + delete ptr; + } + } + } + } + + static constexpr bool is_page_aligned(uintptr_t addr) noexcept { return (addr & (page_size - 1)) == 0; } static bool is_range_aligned(uintptr_t addr) noexcept @@ -252,6 +276,13 @@ class Page_table { static Pflag flags_of(uintptr_t entry) noexcept { return static_cast(entry & allowed_flags); } + bool has_flag(uintptr_t entry, Pflag fl) noexcept + { + auto* ent = entry_r(entry); + if (ent == nullptr) return false; + return util::has_flag(flags_of(*ent), fl); + } + uintptr_t start_addr() const { return linear_addr_start_; } @@ -270,9 +301,10 @@ class Page_table { } - Summary summary() + Summary summary(bool print = false, int print_lvl = 0) { Summary sum; + const char* pad = "| | | | | | | | | | | | | |"; for (auto& ent : tbl_) { if (is_page(ent) and ((flags_of(ent) & Pflag::present) != Pflag::none)){ sum.add_page(this->page_size); @@ -283,9 +315,19 @@ class Page_table { sum.add_dir(this->page_size); auto* sub = page_dir(&ent); Expects(sub != nullptr); - sum += sub->template summary(); + if (print) { + printf("%.*s-+<%s> 0x%zx\n", print_lvl * 2, pad, + util::Byte_r(page_size).to_string().c_str(), (void*)sub->start_addr()); + } + sum += sub->template summary(print, print_lvl + 1); } } + if (print) { + if (print_lvl) + printf("%.*so\n", (print_lvl * 2) - 1 , pad); + else + printf("o\n"); + } return sum; } @@ -314,24 +356,74 @@ class Page_table { * Allocate a page directory for memory range starting at addr. * applying flags + present **/ - Subdir* create_page_dir(uintptr_t addr, Pflag flags) + Subdir* create_page_dir(uintptr_t lin, uintptr_t phys, Pflag flags = Pflag::none) { - Expects(is_page_aligned(addr)); - auto* entry_ = entry(addr); + Expects(is_page_aligned(lin)); + Expects(is_page_aligned(phys)); + auto* entry_ = entry(lin); Expects(entry_ != nullptr); // Allocate entry - Subdir* sub = allocate_pdir(addr, flags); + Subdir* sub; + sub = new Subdir(lin, phys, flags); + Expects(sub == sub->data()); + Expects(util::bits::is_aligned<4_KiB>(sub)); *entry_ = reinterpret_cast(sub) | (flags & ~Pflag::huge) | Pflag::present | Pflag::pdir; return sub; } + ssize_t bytes_allocated() + { + ssize_t bytes = sizeof(Page_table); + if constexpr (not std::is_void_v) + { + for (auto& ent : tbl_) { + if (is_page_dir(ent)) { + Subdir* ptr = page_dir(&ent); + bytes += ptr->bytes_allocated(); + } + } + } + return bytes; + } + + bool is_empty() + { + for (auto& ent : tbl_) { + if (addr_of(ent) != 0 and this->has_flag(ent, Pflag::present)) + return false; + } + return true; + } + + ssize_t purge_unused() + { + ssize_t bytes = sizeof(Page_table); + if constexpr (not std::is_void_v) + { + for (auto& ent : tbl_) { + if (is_page_dir(ent)) { + Subdir* dir = page_dir(&ent); + if (dir->is_empty()) { + PG_PRINT("Purging %p \n", (void*)addr_of(ent)); + delete dir; + ent = 0; + bytes += sizeof(dir); + } else { + bytes += dir->purge_unused(); + } + } + } + } + return bytes; + } + /** - * Get page dir pointet to by entry. Entry must be created by a previous + * Get page dir pointed to by entry. Entry must be created by a previous * call to create_page_dir **/ Subdir* page_dir(uintptr_t* entry) @@ -376,14 +468,19 @@ class Page_table { **/ void map_all(uintptr_t phys, Pflag flags) { + if (! is_range_aligned(phys)) + PG_PRINT(" Physical addr 0x%lx is not range aligned \n", phys); Expects(is_range_aligned(phys)); for(auto& page : this->tbl_) { - page = phys | flags; + page = phys | flags;; phys += page_size; } } + void id_map(Pflag flags){ + map_all(start_addr(), flags); + } /** * Set flags on a page dir, needed to allow certain flags to have effect @@ -396,8 +493,8 @@ class Page_table { auto curfl = flags_of(*entry); *entry |= flags & allowed_flags & Pflag::permissive; - if (has_flag(curfl, Pflag::no_exec) - && !has_flag(flags, Pflag::no_exec)) { + if (util::has_flag(curfl, Pflag::no_exec) + && ! util::has_flag(flags, Pflag::no_exec)) { *entry &= ~Pflag::no_exec; Ensures((*entry & Pflag::no_exec) == 0); } @@ -409,10 +506,12 @@ class Page_table { { Expects(entry >= tbl_.begin() && entry < tbl_.end()); auto new_entry = addr_of(*entry) | (flags & allowed_flags); + if (!is_page_aligned(addr_of(new_entry))) + printf("%p is not page aligned \n", (void*)addr_of(new_entry)); Expects(is_page_aligned(addr_of(new_entry))); // Can only make pages and page dirs present. E.g. no unmapped PML4 entry. - if (has_flag(flags, Pflag::present) + if (util::has_flag(flags, Pflag::present) and !is_page(new_entry) and !is_page_dir(new_entry)) { PG_PRINT(" Can't set flags on non-aligned entry "); @@ -489,9 +588,13 @@ class Page_table { it != tbl_.end() and res.size < req.size; it++, res.size += page_size) { - *it = req.phys; + if (req.phys != any_addr) { + *it = req.phys; + req.phys += page_size; + } else { + PG_PRINT("Req. any, have: 0x%lx \n", addr_of(*it)); + } set_page_flags(&(*it), req.flags); - req.phys += page_size; } Ensures(res); @@ -500,13 +603,21 @@ class Page_table { return res; } + /** Map a single entry without creating / descending into sub tables **/ Map map_entry(uintptr_t* ent, Map req) { PG_PRINT(" %s\n", req.to_string().c_str()); Expects(ent != nullptr); Expects(ent >= tbl_.begin() && ent < tbl_.end()); Expects(req); - *ent = req.phys; + Expects(not is_page_dir(*ent)); + + if (req.phys != any_addr) { + *ent = req.phys; + } else { + PG_PRINT("Req. any, have: 0x%lx \n", addr_of(*ent)); + } + req.flags = set_page_flags(ent, req.flags); req.size = page_size; req.page_sizes = page_size; @@ -517,8 +628,10 @@ class Page_table { return req; } + Map map_entry_r(uintptr_t* ent, Map req) { + using namespace util; Expects(ent != nullptr); Expects(within_range(req.lin)); @@ -532,14 +645,50 @@ class Page_table { return res; } - // If minimum requested page size was not smaller than this, fail + // If request is an unmap, clear entry, deallocating any allocated page_dir. + bool is_unmap = req.phys == 0 and req.flags == Pflag::none; + if (is_unmap) { + Expects(bits::is_aligned(Arch::min_pagesize, req.size)); + + if (is_page_dir(*ent)) { + auto* pdir = page_dir(ent); + if (req.size >= page_size) { + // Delete this page dir only if it's completely covered by the request + // E.g. Keep it if other mapping with lower page sizes might touch it. + // TODO: Consider checking if the directory is in use, to trigger + // deallocations more frequently. + delete pdir; + } else { + return pdir->map_r(req); + } + } else { + // NOTE: Mapping the 0-page to 0 after id-mapping lower memory implies + // unmap, which again might require creation of new tables. + } + + // Clear the entry if it's definitely not in use after unmap. + if (req.size >= page_size) { + *ent = 0; + req.size = page_size; + return req; + } + } + + // If minimum requested page size was not smaller than this lvl psize, fail if (req.min_psize() >= page_size) return Map(); - // Mapping via sub directory, creating subdir if needed + // Mapping via sub directory, creating subdir if needed. if (!is_page_dir(*ent)) { - auto aligned_addr = req.lin & ~(page_size - 1); - create_page_dir(aligned_addr , flags_of(*ent)); + // Fragment current entry into a new page dir. + + // The requested mapping might not be to the beginning of this entry + auto aligned_addr = req.lin & ~(page_size - 1); + + // Create new page dir, inheriting physical addr and flags from this entry + auto current_addr = addr_of(*ent); + auto current_flags = flags_of(*ent); + create_page_dir(aligned_addr, current_addr, current_flags); } Ensures(is_page_dir(*ent)); @@ -582,14 +731,19 @@ class Page_table { using namespace util; PG_PRINT(" %s\n", req.to_string().c_str()); Expects(req); + Expects(within_range(req.lin)); + Expects(bits::is_aligned(req.lin)); - Expects(bits::is_aligned(req.phys)); Expects(bits::is_aligned(req.min_psize(), req.lin)); - Expects(bits::is_aligned(req.min_psize(), req.phys)); + Expects(req.lin < Arch::max_memory); + + if (req.phys != any_addr) { + Expects(bits::is_aligned(req.phys)); + Expects(bits::is_aligned(req.min_psize(), req.phys)); + Expects(req.phys < Arch::max_memory); + } + Expects((req.page_sizes & os::mem::supported_page_sizes()) != 0); - Expects(req.lin < Platform::max_memory); - Expects(req.phys < Platform::max_memory); - Expects(within_range(req.lin)); Map res{}; @@ -612,7 +766,17 @@ class Page_table { Ensures(res); Ensures((req.page_sizes & res.page_sizes) != 0); Ensures(res.size <= util::bits::roundto<4_KiB>(req.size)); - Ensures(res.lin == req.lin and res.phys == req.phys); + Ensures(res.lin == req.lin); + if (req.phys != any_addr) + Ensures(res.phys == req.phys); + + + if (res.phys == any_addr) { + // TODO: avoid traversing the tables again to get physical addr. + // For now we're using response maps as requests in subsequent calls + // so we can't set it to res.phys deeper in. + res.phys = addr_of(*entry_r(req.lin)); + } return res; } @@ -640,7 +804,7 @@ class Page_table { private: - std::array tbl_ {}; + alignas(Arch::min_pagesize) std::array tbl_ {}; const uintptr_t linear_addr_start_ = 0; }; @@ -703,11 +867,11 @@ inline uintptr_t Pml1::active_page_size(uintptr_t addr) { template <> -inline x86::paging::Summary Pml1::summary(){ +inline x86::paging::Summary Pml1::summary(bool, int){ Summary sum; for (auto ent : tbl_) { - if (has_flag(flags_of(ent), Pflag::present)) + if (util::has_flag(flags_of(ent), Pflag::present)) { sum.pages_4k++; } diff --git a/api/kernel/memory.hpp b/api/kernel/memory.hpp index c954657aab..4c9f64fc49 100644 --- a/api/kernel/memory.hpp +++ b/api/kernel/memory.hpp @@ -39,16 +39,21 @@ namespace mem { uintptr_t supported_page_sizes(); /** Get the smallest supported page size */ - uintptr_t min_psize(); + size_t min_psize(); /** Get the largest supported page size */ - uintptr_t max_psize(); + size_t max_psize(); /** Determine if size is a supported page size */ bool supported_page_size(uintptr_t size); + /** String representation of supported page sizes */ std::string page_sizes_str(size_t bits); + /** + * Virtual to physical memory mapping. + * For interfacing with the virtual memory API, e.g. mem::map / mem::protect. + **/ template struct Mapping { @@ -60,46 +65,26 @@ namespace mem { size_t size = 0; size_t page_sizes = 0; + // Constructors Mapping() = default; - Mapping(uintptr_t linear, uintptr_t physical, Fl fl, size_t sz) - : lin{linear}, phys{physical}, flags{fl}, size{sz}, - page_sizes{any_size} {} - - Mapping(uintptr_t linear, uintptr_t physical, Fl fl, size_t sz, size_t psz) - : lin{linear}, phys{physical}, flags{fl}, size{sz}, page_sizes{psz} - {} - - operator bool() const noexcept - { return size != 0 && page_sizes !=0; } + /** Construct with no page size restrictions */ + inline Mapping(uintptr_t linear, uintptr_t physical, Fl fl, size_t sz); - bool operator==(const Mapping& rhs) const noexcept - { return lin == rhs.lin - && phys == rhs.phys - && flags == rhs.flags - && size == rhs.size - && page_sizes == rhs.page_sizes; } + /** Construct with page size restrictions */ + inline Mapping(uintptr_t linear, uintptr_t physical, Fl fl, size_t sz, size_t psz); - bool operator!=(const Mapping& rhs) const noexcept - { return ! (*this == rhs); } - - inline Mapping operator+(const Mapping& rhs) noexcept; - - Mapping operator+=(const Mapping& rhs) noexcept { - *this = *this + rhs; - return *this; - } - - size_t page_count() const noexcept - { return page_sizes ? (size + page_sizes - 1) / page_sizes : 0; } + inline operator bool() const noexcept; + inline bool operator==(const Mapping& rhs) const noexcept; + inline bool operator!=(const Mapping& rhs) const noexcept; + Mapping operator+(const Mapping& rhs) noexcept; + inline Mapping operator+=(const Mapping& rhs) noexcept; // Smallest page size in map - size_t min_psize() const noexcept - { return util::bits::keepfirst(page_sizes); } + inline size_t min_psize() const noexcept; // Largest page size in map - size_t lmax_psize() const noexcept - { return util::bits::keeplast(page_sizes); } + inline size_t max_psize() const noexcept; std::string to_string() const; @@ -107,6 +92,7 @@ namespace mem { using Map = Mapping<>; + /** Exception class possibly used by various ::mem functions. **/ class Memory_exception : public std::runtime_error { using runtime_error::runtime_error; }; @@ -125,24 +111,46 @@ namespace mem { **/ Map unmap(uintptr_t addr); - /** Get protection flags for page containing a given address */ + /** Get protection flags for page enclosing a given address */ Access flags(uintptr_t addr); /** Determine active page size of a given linear address **/ uintptr_t active_page_size(uintptr_t addr); uintptr_t active_page_size(void* addr); + /** + * Set and return access flags for a given linear address range. + * The range must be a subset of a range mapped by a previous call to map. + * The page sizes will be adjusted to match len as closely as possible, + * creating new page tables as needed. + * Uniform page sizes across the range is not guaranteed unless the enclosing + * range was mapped with a page size restriction. E.g. A len of 2MiB + 4KiB + * might result in 513 4KiB pages or 1 2MiB page and 1 4KiB page getting + * protected. + **/ + Map protect(uintptr_t linear, size_t len, Access flags = Access::read); + /** * Set and return access flags for a given linear address range * The range is expected to be mapped by a previous call to map. **/ - Access protect(uintptr_t linear, Access flags = Access::read); + Access protect_range(uintptr_t linear, Access flags = Access::read); - /** Set and return access flags for a page starting at linear **/ + /** + * Set and return access flags for a page starting at linear. + * @note : the page size can be any of the supported sizes and + * protection will apply for that whole page. + **/ Access protect_page(uintptr_t linear, Access flags = Access::read); + + /** Get the physical address to which linear address is mapped **/ + uintptr_t virt_to_phys(uintptr_t linear); + }} // os::mem + +// Enable bitwise ops on access flags namespace util { inline namespace bitops { template<> @@ -157,12 +165,43 @@ inline namespace bitops { namespace os { namespace mem { + // + // mem::Mapping implementation + // + + template + Mapping::Mapping(uintptr_t linear, uintptr_t physical, Fl fl, size_t sz) + : lin{linear}, phys{physical}, flags{fl}, size{sz}, + page_sizes{any_size} {} + + template + Mapping::Mapping(uintptr_t linear, uintptr_t physical, Fl fl, size_t sz, size_t psz) + : lin{linear}, phys{physical}, flags{fl}, size{sz}, page_sizes{psz} + {} + + template + bool Mapping::operator==(const Mapping& rhs) const noexcept + { return lin == rhs.lin + && phys == rhs.phys + && flags == rhs.flags + && size == rhs.size + && page_sizes == rhs.page_sizes; } + + template + Mapping::operator bool() const noexcept + { return size != 0 && page_sizes !=0; } + + template + bool Mapping::operator!=(const Mapping& rhs) const noexcept + { return ! (*this == rhs); } + template Mapping Mapping::operator+(const Mapping& rhs) noexcept { using namespace util::bitops; Mapping res; + // Adding with empty map behaves like 0 + x / x + 0. if (! rhs) { return *this; } @@ -173,20 +212,20 @@ namespace mem { if (res == rhs) return res; - res.lin = std::min(lin, rhs.lin); - res.phys = std::min(phys, rhs.phys); - // The mappings must have connecting ranges - if ((rhs and rhs.lin + rhs.size != lin) - and (*this and lin + size != rhs.lin)) + if ((rhs.lin + rhs.size != lin) + and lin + size != rhs.lin) { Ensures(!res); return res; } - res.page_sizes |= rhs.page_sizes; + // You can add to the front or the back + res.lin = std::min(lin, rhs.lin); + res.phys = std::min(phys, rhs.phys); // The mappings can span several page sizes + res.page_sizes |= rhs.page_sizes; if (page_sizes && page_sizes != rhs.page_sizes) { res.page_sizes |= page_sizes; @@ -201,6 +240,20 @@ namespace mem { return res; } + template + Mapping Mapping::operator+=(const Mapping& rhs) noexcept { + *this = *this + rhs; + return *this; + } + + template + size_t Mapping::min_psize() const noexcept + { return util::bits::keepfirst(page_sizes); } + + template + size_t Mapping::max_psize() const noexcept + { return util::bits::keeplast(page_sizes); } + template inline std::string Mapping::to_string() const { @@ -215,10 +268,10 @@ namespace mem { const bool isseq = __builtin_popcount(page_sizes) == 1; if (isseq) { - len += snprintf(buffer + len, sizeof(buffer) - len, - " (%lu pages á %s)", - page_count(), - util::Byte_r(page_sizes).to_string().c_str()); + len += snprintf(buffer + len, sizeof(buffer) - len, + " (%lu pages á %s)", + size / page_sizes, + util::Byte_r(page_sizes).to_string().c_str()); } else { len += snprintf(buffer + len, sizeof(buffer) - len, diff --git a/src/arch/x86_64/paging.cpp b/src/arch/x86_64/paging.cpp index ad95973c1f..fedd27ef9e 100644 --- a/src/arch/x86_64/paging.cpp +++ b/src/arch/x86_64/paging.cpp @@ -18,9 +18,11 @@ #include #include #include +#include +// #define DEBUG_X86_PAGING #ifdef DEBUG_X86_PAGING -#define MEM_PRINT(X, ...) printf(" " X, __VA_ARGS__) +#define MEM_PRINT(X, ...) kprintf(" " X, __VA_ARGS__) #else #define MEM_PRINT(X, ...) #endif @@ -39,6 +41,12 @@ using Pml4 = x86::paging::Pml4; static void allow_executable(); +// must be public symbols because of a unittest +extern char _TEXT_START_; +extern char _EXEC_END_; +uintptr_t __exec_begin = (uintptr_t)&_TEXT_START_; +uintptr_t __exec_end = (uintptr_t)&_EXEC_END_; + /** IncludeOS default paging setup @@ -73,14 +81,18 @@ Pml4* __pml4; __attribute__((weak)) void __arch_init_paging() { INFO("x86_64", "Initializing paging"); - __pml4 = x86::paging::allocate_pdir(); + auto default_fl = Flags::present | Flags::writable | Flags::huge | Flags::no_exec; + __pml4 = new Pml4(0); Expects(__pml4 != nullptr); + Expects(!__pml4->has_flag(0, Flags::present)); INFO2("* Supported page sizes: %s", os::mem::page_sizes_str(os::mem::Map::any_size).c_str()); - auto default_fl = Flags::present | Flags::writable | Flags::huge | Flags::no_exec; INFO2("* Adding 512 1GiB entries @ 0x0 -> 0x%llx", 512_GiB); - /* pml3_0 = */ __pml4->create_page_dir(0, default_fl); + auto* pml3_0 = __pml4->create_page_dir(0, 0, default_fl); + + Expects(__pml4->has_flag(0, Flags::present)); + Expects(__pml4->has_flag((uintptr_t)pml3_0, Flags::present)); if (not os::mem::supported_page_size(1_GiB)) { auto first_range = __pml4->map_r({0,0,default_fl, 16_GiB}); @@ -90,29 +102,23 @@ void __arch_init_paging() { INFO2("* Marking page 0 as not present"); auto zero_page = __pml4->map_r({0, 0, Flags::none, 4_KiB, 4_KiB}); - + Expects(__pml4->has_flag(8_KiB, Flags::present | Flags::writable | Flags::no_exec)); allow_executable(); Expects(zero_page.size == 4_KiB); Expects(zero_page.page_sizes == 4_KiB); Expects(__pml4->active_page_size(0LU) == 4_KiB); + Expects(! __pml4->has_flag(0, Flags::present)); + + Expects(! __pml4->has_flag((uintptr_t)__exec_begin, Flags::no_exec)); + Expects(__pml4->has_flag((uintptr_t)__exec_begin, Flags::present)); + INFO2("* Passing page tables to CPU"); extern void __x86_init_paging(void*); __x86_init_paging(__pml4->data()); } -void print_entry(uintptr_t ent) -{ - using namespace x86::paging; - auto flags = __pml4->flags_of(ent); - auto addr = __pml4->addr_of(ent); - auto as_byte = Byte_r(addr); - std::cout << as_byte << std::hex << " 0x" << ent - << " Flags: " << flags << "\n"; -} - - namespace x86 { namespace paging { @@ -143,6 +149,8 @@ Flags to_x86(os::mem::Access prot) Flags flags = Flags::none; if (prot != Access::none) { flags |= Flags::present; + } else { + return Flags::none; } if (has_flag(prot, Access::write)) { @@ -172,8 +180,12 @@ using Map_x86 = Mapping; throw Memory_exception(std::string(__FILE__) + ":" + std::to_string(__LINE__) \ + ": " + reason) +} // mem +} // os + +using namespace os; -uintptr_t supported_page_sizes() { +uintptr_t mem::supported_page_sizes() { static size_t res = 0; if (res == 0) { res = 4_KiB | 2_MiB; @@ -183,17 +195,17 @@ uintptr_t supported_page_sizes() { return res; } -uintptr_t min_psize() +uintptr_t mem::min_psize() { return bits::keepfirst(supported_page_sizes()); } -uintptr_t max_psize() +uintptr_t mem::max_psize() { return bits::keeplast(supported_page_sizes()); } -bool supported_page_size(uintptr_t size) +bool mem::supported_page_size(uintptr_t size) { return bits::is_pow2(size) and (size & supported_page_sizes()) != 0; } @@ -206,7 +218,14 @@ Map_x86 to_x86(Map map){ return {map.lin, map.phys, x86::paging::to_x86(map.flags), map.size, map.page_sizes}; } -Access protect_page(uintptr_t linear, Access flags) +uintptr_t mem::virt_to_phys(uintptr_t linear) +{ + auto* ent = __pml4->entry_r(linear); + if (ent == nullptr) return 0; + return __pml4->addr_of(*ent); +} + +Access mem::protect_page(uintptr_t linear, Access flags) { MEM_PRINT("::protect_page 0x%lx\n", linear); x86::paging::Flags xflags = x86::paging::to_x86(flags); @@ -215,7 +234,7 @@ Access protect_page(uintptr_t linear, Access flags) return to_memflags(f); }; -Access protect(uintptr_t linear, Access flags) +Access mem::protect_range(uintptr_t linear, Access flags) { MEM_PRINT("::protect 0x%lx \n", linear); x86::paging::Flags xflags = x86::paging::to_x86(flags); @@ -227,7 +246,7 @@ Access protect(uintptr_t linear, Access flags) MEM_PRINT("Found entry: %s\n", map_ent.to_string().c_str()); int sz_prot = 0; - x86::paging::Flags fl; + x86::paging::Flags fl{}; // TOOD: Optimize. No need to re-traverse for each page // set_flags_r should probably just take size. @@ -242,17 +261,41 @@ Access protect(uintptr_t linear, Access flags) return to_memflags(fl); }; -Access flags(uintptr_t addr) +Map mem::protect(uintptr_t linear, size_t len, Access flags) +{ + if (UNLIKELY(len < min_psize())) + mem_fail_fast("Can't map less than a page\n"); + + if (UNLIKELY(linear == 0)) + mem_fail_fast("Can't map to address 0"); + + MEM_PRINT("::protect 0x%lx \n", linear); + auto key = OS::memory_map().in_range(linear); + MEM_PRINT("Found key: 0x%zx\n", key); + // Throws if entry wasn't previously mapped. + auto map_ent = OS::memory_map().at(key); + MEM_PRINT("Found entry: %s\n", map_ent.to_string().c_str()); + + auto xflags = x86::paging::to_x86(flags); + x86::paging::Map m{linear, x86::paging::any_addr, xflags, static_cast(len)}; + MEM_PRINT("Wants: %s \n", m.to_string().c_str()); + + Expects(m); + auto res = __pml4->map_r(m); + return to_mmap(res); +} + +Access mem::flags(uintptr_t addr) { return to_memflags(__pml4->flags_r(addr)); } __attribute__((weak)) -Map map(Map m, const char* name) +Map mem::map(Map m, const char* name) { using namespace x86::paging; using namespace util; - + MEM_PRINT("Wants : %s size: %li \n", m.to_string().c_str(), m.size); if (UNLIKELY(!m)) mem_fail_fast("Provided map was empty"); @@ -273,12 +316,14 @@ Map map(Map m, const char* name) if (strcmp(name, "") == 0) name = "mem::map"; - OS::memory_map().assign_range({m.lin, m.lin + m.size - 1, name}); + // Align size to minimal page size; + auto req_addr_end = m.lin + bits::roundto(m.min_psize(), m.size) - 1; + + OS::memory_map().assign_range({m.lin, req_addr_end, name}); auto new_map = __pml4->map_r(to_x86(m)); if (new_map) { - MEM_PRINT("Wants : %s size: %li", m.to_string().c_str(), m.size); - MEM_PRINT("Gets : %s size: %li", new_map.to_string().c_str(), new_map.size); + MEM_PRINT("Gets : %s size: %li\n", new_map.to_string().c_str(), new_map.size); // Size should match requested size rounded up to smallest requested page size Ensures(new_map.size == bits::roundto(m.min_psize(), m.size)); @@ -289,10 +334,11 @@ Map map(Map m, const char* name) return to_mmap(new_map); }; -Map unmap(uintptr_t lin){ +Map mem::unmap(uintptr_t lin){ auto key = OS::memory_map().in_range(lin); Map_x86 m; if (key) { + MEM_PRINT("mem::unmap %p \n", (void*)lin); auto map_ent = OS::memory_map().at(key); m.lin = lin; m.phys = 0; @@ -307,17 +353,11 @@ Map unmap(uintptr_t lin){ return to_mmap(m); } -uintptr_t active_page_size(uintptr_t addr){ +uintptr_t mem::active_page_size(uintptr_t addr){ return __pml4->active_page_size(addr); } -}} // os::mem -// must be public symbols because of a unittest... -extern char _TEXT_START_; -extern char _EXEC_END_; -uintptr_t __exec_begin = (uintptr_t)&_TEXT_START_; -uintptr_t __exec_end = (uintptr_t)&_EXEC_END_; void allow_executable() { diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 8eafe53e6a..3c08c8aa43 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -192,7 +192,7 @@ void OS::legacy_boot() { // Fetch CMOS memory info (unfortunately this is maximally 10^16 kb) auto mem = x86::CMOS::meminfo(); - if (OS::memory_end_ == 0) + if (OS::memory_end_ == __arch_max_canonical_addr) { //uintptr_t low_memory_size = mem.base.total * 1024; INFO2("* Low memory: %i Kib", mem.base.total); diff --git a/test/kernel/integration/paging/service.cpp b/test/kernel/integration/paging/service.cpp index 03c5d3ee05..14ee357dce 100644 --- a/test/kernel/integration/paging/service.cpp +++ b/test/kernel/integration/paging/service.cpp @@ -427,7 +427,7 @@ int main() std::cout << "Protection fault test setup\n"; std::cout << "* Mapping protected page @ " << prot << "\n"; mapped = mem::map(prot, "Protected test page"); - mem::protect((uint64_t)protected_page, mem::Access::read | mem::Access::write); + mem::protect_range((uint64_t)protected_page, mem::Access::read | mem::Access::write); Expects(mapped && mapped == prot); } @@ -443,7 +443,7 @@ int main() pml1 = pml2->page_dir(pml2->entry(mapped.lin)); protected_page[magic->i] = 'a'; - mem::protect((uint64_t)protected_page, mem::Access::read); + mem::protect_range((uint64_t)protected_page, mem::Access::read); Expects(protected_page[magic->i] == 'a'); std::cout << "* Writing to write-protected page, expecting page write fail\n\n"; protected_page[magic->i] = 'b'; @@ -460,7 +460,7 @@ int main() // Read-protect (e.g. not present) std::cout << "* Reading non-present page, expecting page read fail\n\n"; - mem::protect((uint64_t)protected_page, mem::Access::none); + mem::protect_range((uint64_t)protected_page, mem::Access::none); Expects(protected_page[magic->i] == 'b'); } @@ -474,7 +474,7 @@ int main() // Execute protected page std::cout << "* Executing code from execute-protected page, expecting instruction fetch fail\n\n"; - mem::protect((uint64_t)protected_page, mem::Access::read); + mem::protect_range((uint64_t)protected_page, mem::Access::read); ((void(*)())(&protected_page[magic->i]))(); } diff --git a/test/kernel/unit/memory.cpp b/test/kernel/unit/memory.cpp index ec388ed26c..f0989df3e2 100644 --- a/test/kernel/unit/memory.cpp +++ b/test/kernel/unit/memory.cpp @@ -14,11 +14,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#define DEBUG_UNIT +// #define DEBUG_UNIT +#ifdef DEBUG_UNIT +#define MYINFO(X,...) INFO("", X, ##__VA_ARGS__) +#else +#define MYINFO(X,...) +#endif #include #include #include +#include +#include #include using namespace os; @@ -37,23 +44,14 @@ CASE ("Using os::mem::Mapping class") m.page_sizes = 4_KiB; EXPECT(m); - // Correct page count is calculated - EXPECT(m.page_count() == 1); - m.size = 4097; - EXPECT(m.page_count() == 2); + m.page_sizes = 0; + EXPECT(!m); - for (int i = 0; i < 10; i++) { - m.size = rand(); - auto cnt = (m.size / m.page_sizes) - + (m.size % m.page_sizes ? 1 : 0); - EXPECT(m.page_count() == cnt); - } } CASE("os::mem::Mapping Addition") { - mem::Map m{}, n{}; const mem::Map zero{}; EXPECT(!(m + n)); @@ -112,22 +110,42 @@ CASE ("os::mem - Trying to map the 0 page") } extern void __arch_init_paging(); -extern void* __pml4; +extern x86::paging::Pml4* __pml4; + +// Default page setup RAII +class Default_paging { +public: + ~Default_paging() + { + clear_paging(); + } -static void init_default_paging() -{ - if (__pml4 == nullptr) { - free(__pml4); + Default_paging() + { + clear_paging(); + MYINFO("Initializing default paging \n"); __arch_init_paging(); } -} + +private: + static void clear_paging() { + using namespace x86::paging; + MYINFO("Clearing default paging \n"); + if (__pml4 != nullptr) { + __pml4->~Pml4(); + free(__pml4); + __pml4 = nullptr; + OS::memory_map().clear(); + } + } +}; CASE ("os::mem Using map and unmap") { using namespace util; - init_default_paging(); + Default_paging p{}; auto initial_entries = OS::memory_map().map(); // Create a desired mapping @@ -202,12 +220,12 @@ CASE ("os::mem Using map and unmap") } -CASE ("os::mem using protect and flags") +CASE ("os::mem using protect_range and flags") { using namespace util; + Default_paging p{}; - init_default_paging(); EXPECT(__pml4 != nullptr); mem::Map req = {6_GiB, 3_GiB, mem::Access::read, 15 * 4_KiB, 4_KiB}; auto previous_flags = mem::flags(req.lin); @@ -220,7 +238,7 @@ CASE ("os::mem using protect and flags") auto page_above = req.lin + 4_KiB; mem::protect_page(page_below, mem::Access::none); EXPECT(mem::flags(page_below) == mem::Access::none); - EXPECT(mem::active_page_size(page_below) == 2_MiB); + EXPECT(mem::active_page_size(page_below) >= 2_MiB); mem::protect_page(page_above, mem::Access::none); EXPECT(mem::flags(page_above) == mem::Access::none); @@ -232,7 +250,7 @@ CASE ("os::mem using protect and flags") // Can't protect a range that isn't mapped auto unmapped = 590_GiB; EXPECT(mem::flags(unmapped) == mem::Access::none); - EXPECT_THROWS(mem::protect(unmapped, mem::Access::write | mem::Access::read)); + EXPECT_THROWS(mem::protect_range(unmapped, mem::Access::write | mem::Access::read)); EXPECT(mem::flags(unmapped) == mem::Access::none); // You can still protect page @@ -242,7 +260,7 @@ CASE ("os::mem using protect and flags") // But a 512 GiB page can't be present without being mapped EXPECT(mem::protect_page(unmapped, rw) == mem::Access::none); - mem::protect(req.lin, mem::Access::execute); + mem::protect_range(req.lin, mem::Access::execute); for (auto p = req.lin; p < req.lin + req.size; p += 4_KiB){ EXPECT(mem::active_page_size(p) == 4_KiB); EXPECT(mem::flags(p) == (mem::Access::execute | mem::Access::read)); @@ -250,3 +268,168 @@ CASE ("os::mem using protect and flags") EXPECT(mem::flags(req.lin + req.size) == previous_flags); } + + +CASE ("os::mem page table destructors") +{ + using namespace util; + using namespace x86::paging; + + // Running this test with valgrind should show 0 bytes leaked. + Default_paging p{}; + + auto* ptr = new Pml4(0); + EXPECT((uintptr_t(ptr) & ~(0x1000 - 1)) == uintptr_t(ptr)); + EXPECT(ptr->is_empty()); + EXPECT(ptr->bytes_allocated() == sizeof(Pml4)); + delete ptr; + + std::array ints {}; + std::array tbls {}; + + Pml4* ars = new Pml4[8]; + delete[] ars; + + EXPECT(sizeof(Pml4) <= 4096 * 2); + EXPECT(sizeof(ars) <= 8 * sizeof(Pml4)); + std::array stack_tbls{{0}}; + + int it = 0; + for (auto& tbl : tbls) + { + tbl = new Pml4(0); + EXPECT(bits::is_aligned(tbl)); + } + + for (auto& tbl : tbls) delete tbl; + + +} + + +CASE ("os::mem using protect") +{ +SETUP ("Assuming a default page table setup") +{ + using namespace util; + Default_paging p{}; + + EXPECT(__pml4 != nullptr); + + SECTION ("os::mem protect basics") + { + // Save some page sizes around the area we'll be mapping + std::array sizes_pre = {mem::active_page_size(5_GiB), + mem::active_page_size(6_GiB), + mem::active_page_size(6_GiB + 100_MiB), + mem::active_page_size(7_GiB)}; + + // You can't protect an unmapped range + EXPECT_THROWS(mem::protect(6_GiB + 900_MiB, 300_MiB, mem::Access::read)); + + // Map something (a lot will be mapped by default in IncludeOS) + mem::Map req {6_GiB, 3_GiB, mem::Access::read | mem::Access::write, 300_MiB}; + auto res = mem::map(req); + EXPECT(res); + EXPECT(res.flags == (mem::Access::write | mem::Access::read)); + EXPECT(res.lin == req.lin); + EXPECT(res.phys == req.phys); + + // You can't protect a partially mapped range + EXPECT_THROWS(mem::protect(5_GiB + 900_MiB, 300_MiB, mem::Access::read | mem::Access::write)); + auto prot_offs = 100_MiB; + auto prot_begin = req.lin + prot_offs; + auto prot_size = 12_KiB; + + EXPECT(mem::virt_to_phys(req.lin) == req.phys); + EXPECT(mem::virt_to_phys(prot_begin) == req.phys + prot_offs); + + // You can protect a subset of a mapped range + auto pres = mem::protect(prot_begin, prot_size); + + EXPECT(pres); + EXPECT(pres.flags == mem::Access::read); + EXPECT(pres.lin == prot_begin); + EXPECT(pres.size == prot_size); + EXPECT(pres.page_sizes == 4_KiB); + EXPECT(pres.phys == res.phys + prot_offs); + + uintptr_t p = pres.phys; + // The new range will have the smallest page size + for (uintptr_t v = prot_begin; v < prot_begin + prot_size; v += 4_KiB, p += 4_KiB) + { + EXPECT(mem::active_page_size(v) == 4_KiB); + EXPECT(mem::virt_to_phys(v) == p); + } + + // The rest of memory is largely as before except adjacent areas + std::array sizes_post = {mem::active_page_size(5_GiB), + mem::active_page_size(6_GiB), + mem::active_page_size(prot_begin), + mem::active_page_size(7_GiB)}; + + EXPECT(sizes_pre[0] == sizes_post[0]); // Can't have changed, 1 GiB below + EXPECT(sizes_pre[1] >= sizes_post[1]); // Is now likely 2_MiB or 4_KiB + EXPECT(sizes_pre[2] > sizes_post[2]); // Must be the new size + EXPECT(sizes_pre[3] == sizes_post[3]); // Can't have changed, 1 GiB above + + } +} +} + +CASE("os::mem::protect try to break stuff"){ + using namespace util::literals; + auto init_access = mem::Access::none; + + EXPECT(__pml4 == nullptr); + __pml4 = new x86::paging::Pml4(0); + EXPECT(__pml4->is_empty()); + auto initial_use = __pml4->bytes_allocated(); + MYINFO("Initial memory use: %zi \n", initial_use); + + for (auto r : test::random) { + auto lin = 3 % 2 ? (uintptr_t)r % (1_GiB) : (uintptr_t)r % 2_MiB; + auto phys = 1_MiB + r % 100_MiB; + auto size = 4_KiB + (r % 2 ? r % 2_GiB : r % 4_MiB); + + mem::Map req; + req.lin = util::bits::roundto<4_KiB>(lin); + req.phys = util::bits::roundto<4_KiB>(phys); + req.flags = mem::Access::none; + req.size = util::bits::roundto<4_KiB>(size); + req.page_sizes = mem::Map::any_size; + + if (r % 3 == 0) + req.flags |= mem::Access::read; + if (r % 3 == 1) + req.flags |= mem::Access::write; + if (r % 3 == 2) + req.flags |= mem::Access::execute; + + auto m = mem::map(req); + EXPECT(m); + auto bytes_after_map = __pml4->bytes_allocated(); + EXPECT(bytes_after_map > initial_use); + + MYINFO("Allocated bytes after map: %zi == %zi tables\n", + bytes_after_map, bytes_after_map / sizeof(decltype(*__pml4))); + + // Unmap + mem::unmap(m.lin); + EXPECT(__pml4->bytes_allocated() <= bytes_after_map); + auto bytes_after_unmap = __pml4->bytes_allocated(); + MYINFO("Allocated bytes after unmap: %zi == %zi tables\n", + bytes_after_unmap, bytes_after_unmap / sizeof(decltype(*__pml4))); + + // Purge unused + __pml4->purge_unused(); + + auto bytes_after_purge = __pml4->bytes_allocated(); + MYINFO("Allocated bytes after purge: %zi == %zi tables\n", + bytes_after_purge, bytes_after_purge / sizeof(decltype(*__pml4))); + + EXPECT(__pml4->bytes_allocated() == initial_use); + } + MYINFO("Allocated bytes at end: %zi \n", __pml4->bytes_allocated()); + delete __pml4; +} diff --git a/test/kernel/unit/x86_paging.cpp b/test/kernel/unit/x86_paging.cpp index 7270929d24..eb14697b41 100644 --- a/test/kernel/unit/x86_paging.cpp +++ b/test/kernel/unit/x86_paging.cpp @@ -199,7 +199,7 @@ CASE("x86::paging 4-level x86_64 paging") { SETUP ("Initializing page tables") { using namespace x86::paging; using Pflag = x86::paging::Flags; - Pml4* __pml4 = allocate_pdir(); + Pml4* __pml4 = new Pml4(); EXPECT(__pml4 != nullptr); EXPECT(__pml4->size() == 512); @@ -338,7 +338,7 @@ void init_default_paging(uintptr_t exec_beg = 0xa00000, uintptr_t exec_end = 0xb // Initialize default paging (all except actually passing it to CPU) if (__pml4 != nullptr) { - free(__pml4); + delete __pml4; OS::memory_map().clear(); } __arch_init_paging(); @@ -566,13 +566,21 @@ CASE ("x86::paging Verify default paging setup") using Tbl = std::array; Tbl& tbl4 = *(Tbl*)__pml4->data(); + + std::cout << __pml4->summary(true, 0) << "\n"; + // PML4 for (auto& ent4 : tbl4 ) { if (ent4 & Flags::pdir) { page_dirs_found++; auto* sub3 = __pml4->page_dir(&ent4); EXPECT(sub3); - EXPECT(sub3->at(0) != 0); + + // Expect the first pml3 to have a mapped 0-entry + if ((void*)&ent4 == (void*)&tbl4) { + EXPECT(sub3->at(0) != 0); + } + EXPECT_THROWS(sub3->at(512)); Tbl& tbl3 = *(Tbl*)sub3->data(); // PML3 From f4fb89971992baee7f5b5c1ccbe53332f302409b Mon Sep 17 00:00:00 2001 From: niks3089 Date: Mon, 14 May 2018 18:27:46 +0530 Subject: [PATCH 261/723] test: Add multicast unit test --- test/net/unit/ip6_addr.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/net/unit/ip6_addr.cpp b/test/net/unit/ip6_addr.cpp index c729723a5a..8718f096d6 100644 --- a/test/net/unit/ip6_addr.cpp +++ b/test/net/unit/ip6_addr.cpp @@ -72,3 +72,14 @@ CASE("Determine if an address is loopback") EXPECT(not l2.is_loopback()); EXPECT(not l3.is_loopback()); } + +CASE("Determine if an address is a multicast") +{ + Addr l1 { 0xffffffff,0,0,1 }; + Addr l2 { 0xff000000,0,0,0}; + Addr l3 { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd }; + + EXPECT(l1.is_multicast()); + EXPECT(l2.is_multicast()); + EXPECT(not l3.is_multicast()); +} From bc36591f6023c3d3b9f87e661f908ed29d635a20 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Mon, 14 May 2018 18:30:22 +0530 Subject: [PATCH 262/723] ip6: Fix prefix print bug --- src/net/ip6/ip6.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index c4ede5671c..82fb98fcf4 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -300,9 +300,9 @@ namespace net // Compare subnets to know where to send packet next_hop = target == local ? packet->ip_dst() : stack_.gateway6(); - PRINT(" Next hop for %s, (netmask %s, local IP: %s, gateway: %s) == %s\n", + PRINT(" Next hop for %s, (netmask %d, local IP: %s, gateway: %s) == %s\n", packet->ip_dst().str().c_str(), - stack_.netmask6().str().c_str(), + stack_.netmask6(), stack_.ip6_addr().str().c_str(), stack_.gateway6().str().c_str(), next_hop.str().c_str()); From e0c75ee9a4eb7766a03d232eaceb6ea626ba4974 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Mon, 14 May 2018 23:17:15 +0200 Subject: [PATCH 263/723] test: update paging integration test --- test/kernel/integration/paging/service.cpp | 16 ---------------- test/kernel/unit/x86_paging.cpp | 2 -- 2 files changed, 18 deletions(-) diff --git a/test/kernel/integration/paging/service.cpp b/test/kernel/integration/paging/service.cpp index 14ee357dce..874bfa6768 100644 --- a/test/kernel/integration/paging/service.cpp +++ b/test/kernel/integration/paging/service.cpp @@ -104,9 +104,6 @@ extern "C" void __cpu_exception(uintptr_t* regs, int error, uint32_t code){ OS::reboot(); } -extern void print_entry(uintptr_t ent); - - template struct enable_log { static constexpr int level = 0; @@ -193,18 +190,6 @@ void verify_test_entries(){ std::vector test_entries { 0_b, 4_KiB, 8_KiB, 1_MiB, 2_MiB, 100_MiB, 1_GiB, 1002_MiB, 500_GiB, 513_GiB, magic_loc}; - for (auto ent : test_entries) { - std::cout << "\nGetting entry 0x" << std::hex << ent << "\n"; - auto leaf_ent = *__pml4->entry_r(ent); - std::cout << "Leaf: "; - print_entry(leaf_ent); - auto pml_ent = *__pml4->entry(ent); - std::cout << "PML4:"; - print_entry(pml_ent); - std::cout << "Active page size: " << Byte_r(__pml4->active_page_size(ent)) << "\n"; - - } - Expects(mem::active_page_size(0LU) == 4_KiB); Expects(mem::active_page_size(4_KiB) == 4_KiB); Expects(mem::active_page_size(200_MiB) == 2_MiB); @@ -281,7 +266,6 @@ void verify_magic() { Expects(m.size == 4_KiB); Expects(m.lin == magic_loc); Expects(m.phys == (uintptr_t)magic_phys); - Expects(m.page_count() == 1); if (magic_phys->id != '!') { *magic = Magic(); diff --git a/test/kernel/unit/x86_paging.cpp b/test/kernel/unit/x86_paging.cpp index eb14697b41..cd46f1d3d7 100644 --- a/test/kernel/unit/x86_paging.cpp +++ b/test/kernel/unit/x86_paging.cpp @@ -240,7 +240,6 @@ CASE("x86::paging 4-level x86_64 paging") { x86::paging::Map m; EXPECT(not m); - EXPECT(m.page_count() == 0); m.lin = 0; m.phys = 4_KiB; // NOTE: Execute is allowed by default on intel @@ -290,7 +289,6 @@ CASE("x86::paging 4-level x86_64 paging") { x86::paging::Map m; EXPECT(not m); - EXPECT(m.page_count() == 0); m.lin = lin; m.phys = phys; m.flags = Pflag::present; From e3cb481e38803a9ff34c3eded1f11aeda0e83919 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Tue, 15 May 2018 10:25:11 +0200 Subject: [PATCH 264/723] test: Changed name of node process to reflect real name --- test/net/integration/microLB/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net/integration/microLB/test.py b/test/net/integration/microLB/test.py index 68487f8a44..a899eae64e 100755 --- a/test/net/integration/microLB/test.py +++ b/test/net/integration/microLB/test.py @@ -19,7 +19,7 @@ def validateRequest(expected = ""): return (response.content) == expected # start nodeJS -pro = subprocess.Popen(["node", "server.js"], stdout=subprocess.PIPE) +pro = subprocess.Popen(["nodejs", "server.js"], stdout=subprocess.PIPE) def startBenchmark(line): print " starting test " From 2ff9a3fd6054b353e1bed223794823bbfa64fdc2 Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Tue, 15 May 2018 11:55:35 +0200 Subject: [PATCH 265/723] cmake: Adding CERTS option (install_certificates). Now possible: cmake .. -DCERTS='disk/certs' --- cmake/post.service.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index f86bfc1fdf..6c2ccd13fd 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -405,6 +405,11 @@ function(install_certificates FOLD) file(COPY ${INSTALL_LOC}/cert_bundle/ DESTINATION ${REL_PATH}) endfunction() +if(CERTS) + message(STATUS "Certs folder set: " ${CERTS}) + install_certificates(${CERTS}) +endif() + if(TARFILE) get_filename_component(TAR_RELPATH "${TARFILE}" REALPATH BASE_DIR "${CMAKE_SOURCE_DIR}") From 18b6e91dbe3426fc33fda9c8c08170e7fd8f8fef Mon Sep 17 00:00:00 2001 From: niks3089 Date: Tue, 15 May 2018 16:36:19 +0530 Subject: [PATCH 266/723] icmp6: Support for adding multiple icmp6 payload & make sequence number and id in the icmp header a struct --- api/net/ip6/packet_icmp6.hpp | 46 ++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index fc3bd78db2..22387e131b 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -31,15 +31,20 @@ namespace net { namespace icmp6 { class Packet { + + struct IdSe { + uint16_t identifier; + uint16_t sequence; + }; + struct Header { Type type; uint8_t code; uint16_t checksum; union { - uint16_t identifier; - uint16_t sequence; - uint32_t reserved; - uint32_t rso_flags; + struct IdSe idse; + uint32_t reserved; + uint32_t rso_flags; }; uint8_t payload[0]; }__attribute__((packed)); @@ -59,6 +64,11 @@ namespace icmp6 { uint8_t next; } __attribute__((packed)); + struct nd_options_header { + uint8_t type; + uint8_t len; + } __attribute__((packed)); + struct RouterSol { uint8_t options[0]; @@ -113,10 +123,10 @@ namespace icmp6 { { return header().checksum; } uint16_t id() const noexcept - { return header().identifier; } + { return header().idse.identifier; } uint16_t sequence() const noexcept - { return header().sequence; } + { return header().idse.sequence; } /** * Where the payload of an ICMP packet starts, calculated from the start of the IP header @@ -127,7 +137,7 @@ namespace icmp6 { { return pckt_->ip_header_len() + header_size(); } Span payload() - { return {&(header().payload[0]), pckt_->data_end() - &(header().payload[0]) }; } + { return {&(header().payload[0 + payload_offset_]), pckt_->data_end() - &(header().payload[0 + payload_offset_]) }; } /** Several ICMP messages require the payload to be the header and 64 bits of the * data of the original datagram @@ -157,10 +167,10 @@ namespace icmp6 { { header().code = c; } void set_id(uint16_t s) noexcept - { header().identifier = s; } + { header().idse.identifier = s; } void set_sequence(uint16_t s) noexcept - { header().sequence = s; } + { header().idse.sequence = s; } void set_reserved(uint32_t s) noexcept { header().reserved = s; } @@ -170,7 +180,7 @@ namespace icmp6 { * (identifier and sequence is not used when pointer is used) */ void set_pointer(uint8_t error) - { header().identifier = error; } + { header().idse.identifier = error; } uint16_t compute_checksum() const noexcept { @@ -240,10 +250,19 @@ namespace icmp6 { void set_neighbor_adv_flag(uint32_t flag) { header().rso_flags = htonl(flag << 28); } + void set_ndp_options_header(uint8_t type, uint8_t len) + { + struct nd_options_header ndo; + ndo.type = type; + ndo.len = len; + set_payload({reinterpret_cast (&ndo), sizeof ndo}); + } + void set_payload(Span new_load) { - pckt_->set_data_end(pckt_->ip_header_len() + header_size() + new_load.size()); + pckt_->set_data_end(pckt_->ip_header_len() + header_size() + payload_offset_ + new_load.size()); memcpy(payload().data(), new_load.data(), payload().size()); + payload_offset_ += payload().size(); } /** Get the underlying IP packet */ @@ -252,12 +271,12 @@ namespace icmp6 { /** Construct from existing packet **/ Packet(IP6::IP_packet_ptr pckt) - : pckt_{ std::move(pckt) } + : pckt_{ std::move(pckt) }, payload_offset_{0} { } /** Provision fresh packet from factory **/ Packet(IP6::IP_packet_factory create) - : pckt_{create(Protocol::ICMPv6)} + : pckt_{create(Protocol::ICMPv6)}, payload_offset_{0} { pckt_->increment_data_end(sizeof(Header)); } @@ -268,6 +287,7 @@ namespace icmp6 { private: IP6::IP_packet_ptr pckt_; + uint16_t payload_offset_; }; } } From 3e9801cb67295338a0e37d09719622f6215d85a5 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Tue, 15 May 2018 16:37:59 +0530 Subject: [PATCH 267/723] icmp6: Correct the payload in neighbor advertisement packet. --- src/net/ip6/icmp6.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/net/ip6/icmp6.cpp b/src/net/ip6/icmp6.cpp index 05e3108915..5d9218bea3 100644 --- a/src/net/ip6/icmp6.cpp +++ b/src/net/ip6/icmp6.cpp @@ -115,8 +115,13 @@ namespace net PRINT(" Transmitting Neighbor adv to %s\n", res.ip().ip_dst().str().c_str()); - // Payload - res.set_payload(req.payload()); + // Insert target link address, ICMP6 option header and our mac address + // TODO: This is hacky. Fix it + MAC::Addr dest_mac("c0:01:0a:00:00:2a"); + // Target link address + res.set_payload({req.payload().data(), 16 }); + res.set_ndp_options_header(0x02, 0x01); + res.set_payload({reinterpret_cast (&dest_mac), 6}); // Add checksum res.set_checksum(); From a01fb9bc8414ad815e9730c70105085e53251dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 16 May 2018 11:23:22 +0200 Subject: [PATCH 268/723] tcp: ACK recv data before any user callback --- src/net/tcp/connection.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 75cc17f3d0..45e5213d3c 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -718,10 +718,13 @@ void Connection::recv_data(const Packet& in) length = res.length; } + // make sure to mark the data as recveied (ACK) before putting in buffer, + // since user callback can result in sending new data, which means we + // want to ACK the data recv at the same time + cb.RCV.NXT += length; const auto recv = read_request->insert(in.seq(), in.tcp_data(), length, in.isset(PSH)); + // this ensures that the data we ACK is actually put in our buffer. Ensures(recv == length); - - cb.RCV.NXT += recv; } // Packet out of order else if((in.seq() - cb.RCV.NXT) < cb.RCV.WND) From bb8321dffa4b963535f90c0b4072ad13a987659f Mon Sep 17 00:00:00 2001 From: niks3089 Date: Wed, 16 May 2018 16:54:11 +0530 Subject: [PATCH 269/723] test: Add test.py to icmp6 integration test. Rename the image to test_icmp6 --- test/net/integration/icmp6/CMakeLists.txt | 4 +- test/net/integration/icmp6/test.py | 50 +++++++++++++++++++++++ test/net/integration/icmp6/vm.json | 2 +- 3 files changed, 53 insertions(+), 3 deletions(-) create mode 100755 test/net/integration/icmp6/test.py diff --git a/test/net/integration/icmp6/CMakeLists.txt b/test/net/integration/icmp6/CMakeLists.txt index c036709709..9e4880bd00 100644 --- a/test/net/integration/icmp6/CMakeLists.txt +++ b/test/net/integration/icmp6/CMakeLists.txt @@ -7,13 +7,13 @@ endif() include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) -project(test_icmp) +project(test_icmp6) # Human-readable name of your service set(SERVICE_NAME "IncludeOS ICMP test") # Name of your service binary -set(BINARY "test_icmp") +set(BINARY "test_icmp6") # Maximum memory can be hard-coded into the binary set(MAX_MEM 128) diff --git a/test/net/integration/icmp6/test.py b/test/net/integration/icmp6/test.py new file mode 100755 index 0000000000..f7f75c5d86 --- /dev/null +++ b/test/net/integration/icmp6/test.py @@ -0,0 +1,50 @@ +#! /usr/bin/env python + +import sys +import os +import subprocess + +includeos_src = os.environ.get('INCLUDEOS_SRC', + os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) +sys.path.insert(0,includeos_src) + +from vmrunner import vmrunner +from vmrunner.prettify import color + +import socket + +# Get an auto-created VM from the vmrunner +vm = vmrunner.vms[0] + +num_successes = 0 + +def start_icmp_test(trigger_line): + global num_successes + + # 1 Ping: Checking output from callback in service.cpp + print color.INFO(""), "Performing ping6 test" + + output_data = "" + for x in range(0, 11): + output_data += vm.readline() + + print output_data + + if "Received packet from gateway" in output_data and \ + "Identifier: 0" in output_data and \ + "Sequence number: 0" in output_data and \ + "Source: fe80:0:0:0:e823:fcff:fef4:83e7" in output_data and \ + "Destination: fe80:0:0:0:e823:fcff:fef4:85bd" in output_data and \ + "Type: ECHO REPLY (129)" in output_data and \ + "Code: DEFAULT (0)" in output_data and \ + "Checksum: " in output_data and \ + "Data: INCLUDEOS12345ABCDEFGHIJKLMNOPQRSTUVWXYZ12345678" in output_data: + num_successes += 1 + print color.INFO(""), "Ping test succeeded" + else: + print color.FAIL(""), "Ping test FAILED" + +vm.on_output("Service IPv4 address: 10.0.0.45, IPv6 address: fe80:0:0:0:e823:fcff:fef4:85bd", start_icmp_test); + +# Boot the VM, taking a timeout as parameter +vm.cmake().boot(50).clean() diff --git a/test/net/integration/icmp6/vm.json b/test/net/integration/icmp6/vm.json index cae9a2a028..2fab306593 100644 --- a/test/net/integration/icmp6/vm.json +++ b/test/net/integration/icmp6/vm.json @@ -1,5 +1,5 @@ { - "image" : "test_icmp.img", + "image" : "test_icmp6.img", "net" : [{"device" : "virtio", "mac" : "c0:01:0a:00:00:2a"}], "mem" : 128, "intrusive" : "True" From cb18cc098d97454ce6cbaa6ba2343d5eb378eace Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 16 May 2018 08:52:05 +0200 Subject: [PATCH 270/723] Test: Setup for tests is now quiet --- test/linux/tcp/callgraph.sh | 2 +- test/net/integration/microLB/setup.sh | 4 ++-- test/net/integration/websocket/setup.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/linux/tcp/callgraph.sh b/test/linux/tcp/callgraph.sh index 5ca9d61a24..4ed8e1114a 100755 --- a/test/linux/tcp/callgraph.sh +++ b/test/linux/tcp/callgraph.sh @@ -2,7 +2,7 @@ echo -e "\n>>> Installing dependencies" -pip install gprof2dot +pip -q install gprof2dot sudo apt install gprof graphviz echo -e "\n>>> Running tcp test service with profiling" diff --git a/test/net/integration/microLB/setup.sh b/test/net/integration/microLB/setup.sh index 6df324e5ed..bb5831ad0e 100755 --- a/test/net/integration/microLB/setup.sh +++ b/test/net/integration/microLB/setup.sh @@ -1,3 +1,3 @@ #!/bin/bash -sudo apt install -y nodejs -sudo pip install requests +sudo apt -qqq install -y nodejs +sudo -H pip -q install requests diff --git a/test/net/integration/websocket/setup.sh b/test/net/integration/websocket/setup.sh index d2f6628c7e..8c4dcc6173 100755 --- a/test/net/integration/websocket/setup.sh +++ b/test/net/integration/websocket/setup.sh @@ -1,3 +1,3 @@ #!/bin/bash set -e -sudo pip install ws4py +sudo -H pip -q install ws4py From ba89eb75474fba1f62e04af2e53c754edc9a1d1c Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 16 May 2018 09:35:21 +0200 Subject: [PATCH 271/723] Test: net/configure resolved ip conflicts with other tests --- test/net/integration/configure/config.json | 6 +++--- test/net/integration/configure/service.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/net/integration/configure/config.json b/test/net/integration/configure/config.json index 00623765dd..9db721d3b5 100644 --- a/test/net/integration/configure/config.json +++ b/test/net/integration/configure/config.json @@ -3,14 +3,14 @@ { "iface": 0, "config": "static", - "address": "10.0.0.42", + "address": "10.0.0.60", "netmask": "255.255.255.0", "gateway": "10.0.0.1" }, { "iface": 1, "config": "static", - "address": "10.0.0.43", + "address": "10.0.0.61", "netmask": "255.255.255.0", "gateway": "10.0.0.1", "dns": "8.8.8.8" @@ -28,7 +28,7 @@ "iface": 5, "config": "dhcp-with-fallback", "timeout": 10, - "address": "10.0.0.44", + "address": "10.0.0.62", "netmask": "255.255.255.0", "gateway": "10.0.0.1" } diff --git a/test/net/integration/configure/service.cpp b/test/net/integration/configure/service.cpp index 74bc489dd3..9a57288d83 100644 --- a/test/net/integration/configure/service.cpp +++ b/test/net/integration/configure/service.cpp @@ -31,7 +31,7 @@ void Service::start() CHECKSERT(stacks[0][0] != nullptr, "eth0 is initialized"); auto& eth0 = *stacks[0][0]; - CHECKSERT(eth0.ip_addr() == ip4::Addr(10,0,0,42), "IP address is 10.0.0.42"); + CHECKSERT(eth0.ip_addr() == ip4::Addr(10,0,0,60), "IP address is 10.0.0.60"); CHECKSERT(eth0.netmask() == ip4::Addr(255,255,255,0), "Netmask is 255.255.255.0"); CHECKSERT(eth0.gateway() == ip4::Addr(10,0,0,1), "Gateway is 10.0.0.1"); CHECKSERT(eth0.dns_addr() == eth0.gateway(), "DNS addr is same as gateway"); @@ -40,7 +40,7 @@ void Service::start() CHECKSERT(stacks[1][0] != nullptr, "eth1 is initialized"); auto& eth1 = *stacks[1][0]; - CHECKSERT(eth1.ip_addr() == ip4::Addr(10,0,0,43), "IP address is 10.0.0.43"); + CHECKSERT(eth1.ip_addr() == ip4::Addr(10,0,0,61), "IP address is 10.0.0.61"); CHECKSERT(eth1.netmask() == ip4::Addr(255,255,255,0), "Netmask is 255.255.255.0"); CHECKSERT(eth1.gateway() == ip4::Addr(10,0,0,1), "Gateway is 10.0.0.1"); CHECKSERT(eth1.dns_addr() == ip4::Addr(8,8,8,8), "DNS addr is 8.8.8.8"); From 51b247c84211491951802d68568ff5eb495cf7d5 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 16 May 2018 13:22:57 +0200 Subject: [PATCH 272/723] Test: Changed ip in transmit test to avoid conflicts --- test/net/integration/transmit/service.cpp | 2 +- test/net/integration/transmit/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/net/integration/transmit/service.cpp b/test/net/integration/transmit/service.cpp index 6f2bfcdc02..fd79ff7d34 100644 --- a/test/net/integration/transmit/service.cpp +++ b/test/net/integration/transmit/service.cpp @@ -27,7 +27,7 @@ auto& timer = hw::PIT::instance(); void Service::start(const std::string&) { static auto& inet = net::Inet::ifconfig<0>( - { 10,0,0,45 }, // IP + { 10,0,0,49 }, // IP { 255,255,255,0 }, // Netmask { 10,0,0,1 }, // Gateway { 8,8,8,8 }); // DNS diff --git a/test/net/integration/transmit/test.py b/test/net/integration/transmit/test.py index adffc9e8ca..999bac8f09 100755 --- a/test/net/integration/transmit/test.py +++ b/test/net/integration/transmit/test.py @@ -12,7 +12,7 @@ def transmit_test(grgr): print " Performing transmit tests" - HOST, PORT = "10.0.0.45", 4242 + HOST, PORT = "10.0.0.49", 4242 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) data = "Someone there?" From 478498723b31b332c4a78f389ca33454cab29f43 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 16 May 2018 13:27:02 +0200 Subject: [PATCH 273/723] Test: Changed ip in vlan test to avoid conflicts --- test/net/integration/vlan/config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/net/integration/vlan/config.json b/test/net/integration/vlan/config.json index 94e1c28778..68c4b10886 100644 --- a/test/net/integration/vlan/config.json +++ b/test/net/integration/vlan/config.json @@ -3,7 +3,7 @@ { "iface": 0, "config": "static", - "address": "10.0.0.42", + "address": "10.0.0.50", "netmask": "255.255.255.0", "vlan": [ @@ -22,7 +22,7 @@ { "iface": 1, "config": "static", - "address": "10.0.0.43", + "address": "10.0.0.51", "netmask": "255.255.255.0", "vlan": [ From 7b9f7e4a2757c47d466896354ee62e8f2880e35e Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 16 May 2018 13:32:31 +0200 Subject: [PATCH 274/723] Test: Changed ip in unik test to avoid ip conflicts --- test/plugin/integration/unik/service.cpp | 2 +- test/plugin/integration/unik/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/plugin/integration/unik/service.cpp b/test/plugin/integration/unik/service.cpp index 700c5e8e1c..33b5ab7c53 100644 --- a/test/plugin/integration/unik/service.cpp +++ b/test/plugin/integration/unik/service.cpp @@ -36,7 +36,7 @@ void Service::start(const std::string&) CHECKSERT(not net::Inet::stack<0>().udp().is_bound(unik::default_port), "Unik UDP port is free as expected"); INFO("Unik test", "Manual netwok config"); - net::Inet::stack<0>().network_config({10,0,0,42},{255,255,255,0},{10,0,0,1},{8,8,8,8}); + net::Inet::stack<0>().network_config({10,0,0,56},{255,255,255,0},{10,0,0,1},{8,8,8,8}); unik::Client::register_instance(net::Inet::stack<0>()); } else { diff --git a/test/plugin/integration/unik/test.py b/test/plugin/integration/unik/test.py index a5c90fa4ee..fe5253614b 100755 --- a/test/plugin/integration/unik/test.py +++ b/test/plugin/integration/unik/test.py @@ -9,6 +9,6 @@ from vmrunner import vmrunner -# TODO: Implement a mockup of the Unik registration protocol on 10.0.0.42 +# TODO: Implement a mockup of the Unik registration protocol on 10.0.0.56 vmrunner.vms[0].cmake().boot(60).clean() From 35e0805a3a17ddb8e046d44b9d93abeb36620ff5 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 16 May 2018 13:36:58 +0200 Subject: [PATCH 275/723] Test: Moved ip address conflict overview to test README.md --- test/README.md | 30 +++++++++++++++++++++++++++++- test/net/integration/README.md | 10 ---------- 2 files changed, 29 insertions(+), 11 deletions(-) delete mode 100644 test/net/integration/README.md diff --git a/test/README.md b/test/README.md index cd80ad5635..9ef43991a5 100644 --- a/test/README.md +++ b/test/README.md @@ -1,4 +1,32 @@ # Test services -More or less in progress, some work some don't. +More or less in progress, some work some don't. The idea is to have various services here, which test different functionality, and that can be automated. + +## IP addresses in use by test services + +The following IP addresses are used by the services in the test folder. Make sure to use an unused one to avoid collisions when running tests in parallel. + +### Network tests +- bufstore: No IP +- configure: 10.0.0.60, 10.0.0.61, 10.0.0.62 +- dhclient: Time_sensitive, leave for now +- dhcpd: 10.0.0.1, 10.0.0.10, 10.0.0.11, 10.0.0.12 - Is 10.0.0.1 a problem? +- dhcpd_dhclient_linux: leave for now +- dns: 10.0.0.48 +- gateway: 10.0.1.1, 10.0.1.10, 10.0.2.1, 10.0.2.10 +- http: 10.0.0.46 +- icmp: 10.0.0.45 +- ipv6: 10.0.0.47 +- microLB: 10.0.0.1, 10.0.0.68, 10.0.0.69 - Is 10.0.0.1 a problem? +- nat: 10.1.0.1, 192.1.0.1, 10.1.0.10, 192.1.0.192, 10.1.10.20 +- router: Intrusive + time sensitive - runs alone +- tcp: 10.0.0.44 +- transmit: 10.0.0.49 +- udp: 10.0.0.55 +- vlan: 10.0.0.50, 10.50.0.10, 10.60.0.10, 10.0.0.51, 10.50.0.20, 10.60.0.20 +- websocket: 10.0.0.54 + +### Others +- stress: 10.0.0.42 +- plugin/unik: 10.0.0.56 diff --git a/test/net/integration/README.md b/test/net/integration/README.md deleted file mode 100644 index 4dda2e2a55..0000000000 --- a/test/net/integration/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# IP addresses in use by test services - -The following IP addresses are used by the services in the test folder. Make sure to use an unused one to avoid collisions when running tests in parallell. - -- stress: 10.0.0.42 -- transmit: 10.0.0.43 -- tcp: 10.0.0.44 -- udp: 10.0.0.45 -- websocket: 10.0.0.54 -- dhcpd: 10.0.0.1, 10.0.0.10, 10.0.0.20 From 669d79c82f013b05b63e0be7f7eb78ec042afbdc Mon Sep 17 00:00:00 2001 From: niks3089 Date: Wed, 16 May 2018 17:35:21 +0530 Subject: [PATCH 276/723] test: icmp6, remove intrusive keyword and use 10.0.0.51 ip instead --- test/net/integration/icmp6/service.cpp | 2 +- test/net/integration/icmp6/vm.json | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/net/integration/icmp6/service.cpp b/test/net/integration/icmp6/service.cpp index 509e067abe..6d42d1dd9d 100644 --- a/test/net/integration/icmp6/service.cpp +++ b/test/net/integration/icmp6/service.cpp @@ -23,7 +23,7 @@ using namespace net; void Service::start(const std::string&) { auto& inet = Inet::stack<0>(); - inet.network_config({ 10, 0, 0, 45 }, // IP + inet.network_config({ 10, 0, 0, 51 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 1 }, // Gateway { 8, 8, 8, 8 } // DNS diff --git a/test/net/integration/icmp6/vm.json b/test/net/integration/icmp6/vm.json index 2fab306593..d30ddc3513 100644 --- a/test/net/integration/icmp6/vm.json +++ b/test/net/integration/icmp6/vm.json @@ -1,6 +1,5 @@ { "image" : "test_icmp6.img", "net" : [{"device" : "virtio", "mac" : "c0:01:0a:00:00:2a"}], - "mem" : 128, - "intrusive" : "True" + "mem" : 128 } From 1a78e1a591d01af6d5c2d2d8c28f4929c09b61c7 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 16 May 2018 14:27:28 +0200 Subject: [PATCH 277/723] Test: Moved try/except out of else statement. Now it is always run. --- test/plugin/integration/unik/service.cpp | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/test/plugin/integration/unik/service.cpp b/test/plugin/integration/unik/service.cpp index 33b5ab7c53..db87915e49 100644 --- a/test/plugin/integration/unik/service.cpp +++ b/test/plugin/integration/unik/service.cpp @@ -42,13 +42,12 @@ void Service::start(const std::string&) } else { INFO("Unik test", "DHCP OK. We can now use the IP stack"); CHECK(net::Inet::stack<0>().udp().is_bound(unik::default_port), "Unik UDP port is bound as expected"); - try { - net::Inet::stack<0>().udp().bind(unik::default_port); - } catch(net::UDP::Port_in_use_exception& e){ - CHECK(true, "Trying to bound to the Unik port now fails"); - INFO("Unik test", "SUCCESS"); - } + } + try { + net::Inet::stack<0>().udp().bind(unik::default_port); + } catch(net::UDP::Port_in_use_exception& e){ + CHECK(true, "Trying to bound to the Unik port now fails"); + INFO("Unik test", "SUCCESS"); } }); - } From 034a4e81f3da540eb478d77251ea1dda228b7f85 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Wed, 16 May 2018 18:16:41 +0530 Subject: [PATCH 278/723] test: Change the address from 10.0.0.51 to 10.0.0.52 --- test/net/integration/icmp6/service.cpp | 2 +- test/net/integration/icmp6/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/net/integration/icmp6/service.cpp b/test/net/integration/icmp6/service.cpp index 6d42d1dd9d..9884ebe948 100644 --- a/test/net/integration/icmp6/service.cpp +++ b/test/net/integration/icmp6/service.cpp @@ -23,7 +23,7 @@ using namespace net; void Service::start(const std::string&) { auto& inet = Inet::stack<0>(); - inet.network_config({ 10, 0, 0, 51 }, // IP + inet.network_config({ 10, 0, 0, 52 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 1 }, // Gateway { 8, 8, 8, 8 } // DNS diff --git a/test/net/integration/icmp6/test.py b/test/net/integration/icmp6/test.py index f7f75c5d86..5728a5b98e 100755 --- a/test/net/integration/icmp6/test.py +++ b/test/net/integration/icmp6/test.py @@ -44,7 +44,7 @@ def start_icmp_test(trigger_line): else: print color.FAIL(""), "Ping test FAILED" -vm.on_output("Service IPv4 address: 10.0.0.45, IPv6 address: fe80:0:0:0:e823:fcff:fef4:85bd", start_icmp_test); +vm.on_output("Service IPv4 address: 10.0.0.52, IPv6 address: fe80:0:0:0:e823:fcff:fef4:85bd", start_icmp_test); # Boot the VM, taking a timeout as parameter vm.cmake().boot(50).clean() From a1216c0ed7342aaa62cf30bfb9a26665348ff076 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 16 May 2018 14:52:13 +0200 Subject: [PATCH 279/723] Test: Changed ip in modules test to avoid conflicts --- test/kernel/integration/modules/mod2/service.cpp | 2 +- test/kernel/integration/modules/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/kernel/integration/modules/mod2/service.cpp b/test/kernel/integration/modules/mod2/service.cpp index 2dde55da81..60f055d8f6 100644 --- a/test/kernel/integration/modules/mod2/service.cpp +++ b/test/kernel/integration/modules/mod2/service.cpp @@ -21,7 +21,7 @@ void Service::start() { auto& inet = net::Inet::stack<0>(); - inet.network_config({10,0,0,42}, + inet.network_config({10,0,0,53}, {255,255,255,0}, {10,0,0,1}); printf("IncludeOS was just chainloaded by IncludeOS\n"); diff --git a/test/kernel/integration/modules/test.py b/test/kernel/integration/modules/test.py index 1278682d86..a387a1d48f 100755 --- a/test/kernel/integration/modules/test.py +++ b/test/kernel/integration/modules/test.py @@ -4,7 +4,7 @@ import os import subprocess -HOST="10.0.0.42" +HOST="10.0.0.53" includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) sys.path.insert(0,includeos_src) From 0c8e9c449bb902f0ce1d3c74b09fa4163f239a11 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 16 May 2018 14:52:28 +0200 Subject: [PATCH 280/723] Test: Added a few more ips to conflict table in readme. --- test/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/README.md b/test/README.md index 9ef43991a5..546cb50d8f 100644 --- a/test/README.md +++ b/test/README.md @@ -17,6 +17,7 @@ The following IP addresses are used by the services in the test folder. Make sur - gateway: 10.0.1.1, 10.0.1.10, 10.0.2.1, 10.0.2.10 - http: 10.0.0.46 - icmp: 10.0.0.45 +- icmp6: 10.0.0.52 - ipv6: 10.0.0.47 - microLB: 10.0.0.1, 10.0.0.68, 10.0.0.69 - Is 10.0.0.1 a problem? - nat: 10.1.0.1, 192.1.0.1, 10.1.0.10, 192.1.0.192, 10.1.10.20 @@ -30,3 +31,4 @@ The following IP addresses are used by the services in the test folder. Make sur ### Others - stress: 10.0.0.42 - plugin/unik: 10.0.0.56 +- kernel/modules: 10.0.0.53 From 574949a931e844208a66b907bc70c7cadc14b674 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Thu, 17 May 2018 12:34:17 +0530 Subject: [PATCH 281/723] test: Add setup.sh file for icmp6 integration test --- test/net/integration/icmp6/setup.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 test/net/integration/icmp6/setup.sh diff --git a/test/net/integration/icmp6/setup.sh b/test/net/integration/icmp6/setup.sh new file mode 100755 index 0000000000..4e6f22a543 --- /dev/null +++ b/test/net/integration/icmp6/setup.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 From 81b5e0cb977d09e142eaacaa08451b19a41b4d0d Mon Sep 17 00:00:00 2001 From: Anders Schau Knatten Date: Mon, 21 May 2018 20:49:22 +0200 Subject: [PATCH 282/723] CONTRIBUTE.md: Fix link to contribution guidelines The old link was to a wiki that seems to no longer exist. In README.md, I found a link to the contribution guidelines on readthedocs, so I assume that's the one we want to use. --- CONTRIBUTE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index 2d36f1deea..e9a9d7e291 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -2,6 +2,6 @@ You're very welcome to [clone, edit and send pull-request](https://help.github.com/articles/using-pull-requests). -**Please read the contribution gudelines on the wiki: [Conributing to IncludeOS](https://github.com/hioa-cs/IncludeOS/wiki/Contributing-to-IncludeOS)** +**Please read the contribution guidelines: [Contributing to IncludeOS](http://includeos.readthedocs.io/en/latest/Contributing-to-IncludeOS.html)** ## Thank you! From 864905ea00822f21c9cbe4be75356e8783cb3de6 Mon Sep 17 00:00:00 2001 From: Anders Schau Knatten Date: Mon, 21 May 2018 21:24:31 +0200 Subject: [PATCH 283/723] gitignore: Vim and CLion Ignore vim swap files (.swp) and CLion project directories (.idea) --- .gitignore | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.gitignore b/.gitignore index 8d4ba4046b..96eadfbcc6 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,9 @@ cmake_install.cmake # Name of installation folder IncludeOS_install + +# Vim +*.swp + +#CLion +.idea/ From 7ebe3436c509943c28d353fcf1f4c3fd45a3dd6a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 22 May 2018 13:18:40 +0200 Subject: [PATCH 284/723] net: Separate TCP stream from Connection --- api/net/http/connection.hpp | 11 +- api/net/http/response_writer.hpp | 1 - api/net/http/server.hpp | 2 +- api/net/openssl/tls_stream.hpp | 10 +- api/net/stream.hpp | 8 +- api/net/tcp/connection.hpp | 190 ---------------------------- api/net/tcp/stream.hpp | 198 ++++++++++++++++++++++++++++++ api/util/async.hpp | 2 +- lib/LiveUpdate/liveupdate.hpp | 1 + lib/LiveUpdate/serialize_tcp.cpp | 1 + lib/microLB/micro_lb/balancer.cpp | 1 + lib/microLB/micro_lb/openssl.cpp | 1 + src/CMakeLists.txt | 2 +- src/net/http/basic_client.cpp | 2 +- src/net/http/client.cpp | 2 +- src/net/https/botan_server.cpp | 3 +- src/net/https/openssl_server.cpp | 2 +- src/net/tcp/connection.cpp | 13 -- src/net/tcp/stream.cpp | 18 +++ 19 files changed, 245 insertions(+), 223 deletions(-) create mode 100644 api/net/tcp/stream.hpp create mode 100644 src/net/tcp/stream.cpp diff --git a/api/net/http/connection.hpp b/api/net/http/connection.hpp index 9d116c6215..3ab51cf72e 100644 --- a/api/net/http/connection.hpp +++ b/api/net/http/connection.hpp @@ -23,7 +23,7 @@ #include "request.hpp" #include "response.hpp" -#include +#include namespace http { @@ -32,7 +32,7 @@ namespace http { using Stream = net::Stream; using Stream_ptr = std::unique_ptr; using Peer = net::Socket; - using buffer_t = net::tcp::buffer_t; + using buffer_t = net::Stream::buffer_t; public: inline explicit Connection(Stream_ptr stream, bool keep_alive = true); @@ -42,7 +42,7 @@ namespace http { inline explicit Connection() noexcept; - net::tcp::port_t local_port() const noexcept + uint16_t local_port() const noexcept { return (stream_) ? stream_->local().port() : 0; } Peer peer() const noexcept @@ -121,7 +121,7 @@ namespace http { template Connection::Connection(TCP& tcp, Peer addr) - : Connection(std::make_unique(tcp.connect(addr))) + : Connection(std::make_unique(tcp.connect(addr))) { } @@ -142,8 +142,7 @@ namespace http { { auto copy = std::move(stream_); - // this is expensive and may be unecessary, - // but just to be safe for now + // reset delegates before handing out stream copy->reset_callbacks(); return copy; diff --git a/api/net/http/response_writer.hpp b/api/net/http/response_writer.hpp index 68fe85c06d..21bd2912ff 100644 --- a/api/net/http/response_writer.hpp +++ b/api/net/http/response_writer.hpp @@ -41,7 +41,6 @@ namespace http { */ class Response_writer { public: - using TCP_conn = net::tcp::Connection_ptr; using buffer_t = net::tcp::buffer_t; public: diff --git a/api/net/http/server.hpp b/api/net/http/server.hpp index dd1363f5c3..a966e1802c 100644 --- a/api/net/http/server.hpp +++ b/api/net/http/server.hpp @@ -120,7 +120,7 @@ namespace http { * @param[in] conn The TCP connection */ virtual void on_connect(TCP_conn conn) - { connect(std::make_unique(std::move(conn))); } + { connect(std::make_unique(std::move(conn))); } /** * @brief Connect the stream to the server. diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index 5ab5fe4a60..23ace2510a 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #define VERBOSE_OPENSSL #ifdef VERBOSE_OPENSSL @@ -160,12 +160,12 @@ namespace openssl } inline void TLS_stream::write(const std::string& str) { - write(net::tcp::construct_buffer(str.data(), str.data() + str.size())); + write(net::Stream::construct_buffer(str.data(), str.data() + str.size())); } inline void TLS_stream::write(const void* data, const size_t len) { auto* buf = static_cast (data); - write(net::tcp::construct_buffer(buf, buf + len)); + write(net::Stream::construct_buffer(buf, buf + len)); } inline void TLS_stream::tls_read(buffer_t buffer) @@ -217,7 +217,7 @@ namespace openssl char temp[8192]; n = SSL_read(this->m_ssl, temp, sizeof(temp)); if (n > 0) { - auto buf = net::tcp::construct_buffer(temp, temp + n); + auto buf = net::Stream::construct_buffer(temp, temp + n); if (m_on_read) m_on_read(std::move(buf)); } @@ -251,7 +251,7 @@ namespace openssl //printf("pending: %d\n", pending); if (pending > 0) { - auto buffer = net::tcp::construct_buffer(pending); + auto buffer = net::Stream::construct_buffer(pending); int n = BIO_read(this->m_bio_wr, buffer->data(), buffer->size()); assert(n == pending); m_transport->write(buffer); diff --git a/api/net/stream.hpp b/api/net/stream.hpp index 473acb57d5..4ffcc59e11 100644 --- a/api/net/stream.hpp +++ b/api/net/stream.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -37,6 +38,12 @@ namespace net { using buffer_t = std::shared_ptr>; using ptr = Stream_ptr; + /** Construct a shared vector used in TCP **/ + template + buffer_t construct_buffer(Args&&... args) { + return std::make_shared> (std::forward (args)...); + } + /** Called when the stream is ready to be used. */ using ConnectCallback = delegate; /** @@ -186,7 +193,6 @@ namespace net { virtual ~Stream() {} }; // < class Stream - } // < namespace net #endif // < NET_STREAM_HPP diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index f0d1fe826e..b182871445 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -30,7 +30,6 @@ #include #include #include -#include namespace net { // Forward declaration of the TCP object @@ -58,8 +57,6 @@ class Connection { struct Disconnect; /** Reason for packet being dropped */ enum class Drop_reason; - /** A Connection stream */ - class Stream; using Byte = uint8_t; @@ -210,191 +207,6 @@ class Connection { */ inline void abort(); - /** - * @brief Exposes a TCP Connection as a Stream with only the most necessary features. - * May be overrided by extensions like TLS etc for additional functionality. - */ - class Stream : public net::Stream { - public: - /** - * @brief Construct a Stream for a Connection ptr - * - * @param[in] conn The connection - */ - Stream(Connection_ptr conn) - : m_tcp{std::move(conn)} - { - // stream for a nullptr makes no sense - Expects(m_tcp != nullptr); - } - - /** - * @brief Event when the stream is connected/established/ready to use. - * - * @param[in] cb The connect callback - */ - void on_connect(ConnectCallback cb) override - { - m_tcp->on_connect(Connection::ConnectCallback::make_packed( - [this, cb] (Connection_ptr conn) - { if(conn) cb(*this); })); - } - - /** - * @brief Event when data is received. - * - * @param[in] n The size of the receive buffer - * @param[in] cb The read callback - */ - void on_read(size_t n, ReadCallback cb) override - { m_tcp->on_read(n, cb); } - - /** - * @brief Event for when the Stream is being closed. - * - * @param[in] cb The close callback - */ - void on_close(CloseCallback cb) override - { m_tcp->on_close(cb); } - - /** - * @brief Event for when data has been written. - * - * @param[in] cb The write callback - */ - void on_write(WriteCallback cb) override - { m_tcp->on_write(cb); } - - /** - * @brief Async write of a data with a length. - * - * @param[in] buf data - * @param[in] n length - */ - void write(const void* buf, size_t n) override - { m_tcp->write(buf, n); } - - /** - * @brief Async write of a shared buffer with a length. - * - * @param[in] buffer shared buffer - * @param[in] n length - */ - void write(buffer_t buffer) override - { m_tcp->write(buffer); } - - /** - * @brief Async write of a string. - * Calls write(const void* buf, size_t n) - * - * @param[in] str The string - */ - void write(const std::string& str) override - { write(str.data(), str.size()); } - - /** - * @brief Closes the stream. - */ - void close() override - { m_tcp->close(); } - - /** - * @brief Aborts (terminates) the stream. - */ - void abort() override - { m_tcp->abort(); } - - /** - * @brief Resets all callbacks. - */ - void reset_callbacks() override - { m_tcp->reset_callbacks(); } - - /** - * @brief Returns the streams local socket. - * - * @return A TCP Socket - */ - Socket local() const override - { return m_tcp->local(); } - - /** - * @brief Returns the streams remote socket. - * - * @return A TCP Socket - */ - Socket remote() const override - { return m_tcp->remote(); } - - /** - * @brief Returns a string representation of the stream. - * - * @return String representation of the stream. - */ - std::string to_string() const override - { return m_tcp->to_string(); } - - /** - * @brief Determines if connected (established). - * - * @return True if connected, False otherwise. - */ - bool is_connected() const noexcept override - { return m_tcp->is_connected(); } - - /** - * @brief Determines if writable. (write is allowed) - * - * @return True if writable, False otherwise. - */ - bool is_writable() const noexcept override - { return m_tcp->is_writable(); } - - /** - * @brief Determines if readable. (data can be received) - * - * @return True if readable, False otherwise. - */ - bool is_readable() const noexcept override - { return m_tcp->is_readable(); } - - /** - * @brief Determines if closing. - * - * @return True if closing, False otherwise. - */ - bool is_closing() const noexcept override - { return m_tcp->is_closing(); } - - /** - * @brief Determines if closed. - * - * @return True if closed, False otherwise. - */ - bool is_closed() const noexcept override - { return m_tcp->is_closed(); }; - - int get_cpuid() const noexcept override; - - size_t serialize_to(void* p) const override { - return m_tcp->serialize_to(p); - } - - Stream* transport() noexcept override { - return nullptr; - } - - virtual ~Stream() {} - - Connection_ptr tcp() { - return this->m_tcp; - } - - protected: - Connection_ptr m_tcp; - - }; // < class Connection::Stream - /** * @brief Reason for disconnect event. */ @@ -1354,8 +1166,6 @@ class Connection { }; // < class Connection -using Stream = Connection::Stream; - } // < namespace tcp } // < namespace net diff --git a/api/net/tcp/stream.hpp b/api/net/tcp/stream.hpp new file mode 100644 index 0000000000..7276e9134d --- /dev/null +++ b/api/net/tcp/stream.hpp @@ -0,0 +1,198 @@ +#include +#include + +// TCP stream + +namespace net::tcp +{ + /** + * @brief Exposes a TCP Connection as a Stream with only the most necessary features. + * May be overrided by extensions like TLS etc for additional functionality. + */ + class Stream final : public net::Stream { + public: + /** + * @brief Construct a Stream for a Connection ptr + * + * @param[in] conn The connection + */ + Stream(Connection_ptr conn) + : m_tcp{std::move(conn)} + { + // stream for a nullptr makes no sense + Expects(m_tcp != nullptr); + } + ~Stream() + { + this->reset_callbacks(); + this->close(); + } + + /** + * @brief Event when the stream is connected/established/ready to use. + * + * @param[in] cb The connect callback + */ + void on_connect(ConnectCallback cb) override + { + m_tcp->on_connect(Connection::ConnectCallback::make_packed( + [this, cb] (Connection_ptr conn) + { if(conn) cb(*this); })); + } + + /** + * @brief Event when data is received. + * + * @param[in] n The size of the receive buffer + * @param[in] cb The read callback + */ + void on_read(size_t n, ReadCallback cb) override + { m_tcp->on_read(n, cb); } + + /** + * @brief Event for when the Stream is being closed. + * + * @param[in] cb The close callback + */ + void on_close(CloseCallback cb) override + { m_tcp->on_close(cb); } + + /** + * @brief Event for when data has been written. + * + * @param[in] cb The write callback + */ + void on_write(WriteCallback cb) override + { m_tcp->on_write(cb); } + + /** + * @brief Async write of a data with a length. + * + * @param[in] buf data + * @param[in] n length + */ + void write(const void* buf, size_t n) override + { m_tcp->write(buf, n); } + + /** + * @brief Async write of a shared buffer with a length. + * + * @param[in] buffer shared buffer + * @param[in] n length + */ + void write(buffer_t buffer) override + { m_tcp->write(buffer); } + + /** + * @brief Async write of a string. + * Calls write(const void* buf, size_t n) + * + * @param[in] str The string + */ + void write(const std::string& str) override + { write(str.data(), str.size()); } + + /** + * @brief Closes the stream. + */ + void close() override + { + m_tcp->close(); + } + + /** + * @brief Aborts (terminates) the stream. + */ + void abort() override + { m_tcp->abort(); } + + /** + * @brief Resets all callbacks. + */ + void reset_callbacks() override + { m_tcp->reset_callbacks(); } + + /** + * @brief Returns the streams local socket. + * + * @return A TCP Socket + */ + Socket local() const override + { return m_tcp->local(); } + + /** + * @brief Returns the streams remote socket. + * + * @return A TCP Socket + */ + Socket remote() const override + { return m_tcp->remote(); } + + /** + * @brief Returns a string representation of the stream. + * + * @return String representation of the stream. + */ + std::string to_string() const override + { return m_tcp->to_string(); } + + /** + * @brief Determines if connected (established). + * + * @return True if connected, False otherwise. + */ + bool is_connected() const noexcept override + { return m_tcp->is_connected(); } + + /** + * @brief Determines if writable. (write is allowed) + * + * @return True if writable, False otherwise. + */ + bool is_writable() const noexcept override + { return m_tcp->is_writable(); } + + /** + * @brief Determines if readable. (data can be received) + * + * @return True if readable, False otherwise. + */ + bool is_readable() const noexcept override + { return m_tcp->is_readable(); } + + /** + * @brief Determines if closing. + * + * @return True if closing, False otherwise. + */ + bool is_closing() const noexcept override + { return m_tcp->is_closing(); } + + /** + * @brief Determines if closed. + * + * @return True if closed, False otherwise. + */ + bool is_closed() const noexcept override + { return m_tcp->is_closed(); }; + + int get_cpuid() const noexcept override; + + size_t serialize_to(void* p) const override { + return m_tcp->serialize_to(p); + } + + Stream* transport() noexcept override { + return nullptr; + } + + Connection_ptr tcp() { + return this->m_tcp; + } + + protected: + Connection_ptr m_tcp; + + }; // < class Connection::Stream + +} diff --git a/api/util/async.hpp b/api/util/async.hpp index 4bd0feccfd..46c1b8e52e 100644 --- a/api/util/async.hpp +++ b/api/util/async.hpp @@ -20,7 +20,7 @@ #define UTIL_ASYNC_HPP #include -#include +#include #include class Async diff --git a/lib/LiveUpdate/liveupdate.hpp b/lib/LiveUpdate/liveupdate.hpp index 4c1edd0921..95993ef86a 100644 --- a/lib/LiveUpdate/liveupdate.hpp +++ b/lib/LiveUpdate/liveupdate.hpp @@ -23,6 +23,7 @@ #define LIVEUPDATE_HEADER_HPP #include +#include #include #include #include diff --git a/lib/LiveUpdate/serialize_tcp.cpp b/lib/LiveUpdate/serialize_tcp.cpp index 60f7f12384..5de7c26df0 100644 --- a/lib/LiveUpdate/serialize_tcp.cpp +++ b/lib/LiveUpdate/serialize_tcp.cpp @@ -20,6 +20,7 @@ **/ #include #include +#include #include "serialize_tcp.hpp" #include "liveupdate.hpp" #include "storage.hpp" diff --git a/lib/microLB/micro_lb/balancer.cpp b/lib/microLB/micro_lb/balancer.cpp index 26f5d98b0e..099090a253 100644 --- a/lib/microLB/micro_lb/balancer.cpp +++ b/lib/microLB/micro_lb/balancer.cpp @@ -1,4 +1,5 @@ #include "balancer.hpp" +#include #define READQ_PER_CLIENT 4096 #define MAX_READQ_PER_NODE 8192 diff --git a/lib/microLB/micro_lb/openssl.cpp b/lib/microLB/micro_lb/openssl.cpp index 2d0b4ee167..13deb23e2a 100644 --- a/lib/microLB/micro_lb/openssl.cpp +++ b/lib/microLB/micro_lb/openssl.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace microLB { diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 15a101b233..bfaed851b6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -46,7 +46,7 @@ set(OS_OBJECTS net/checksum.cpp net/ip4/arp.cpp net/ip4/ip4.cpp net/ip4/reassembly.cpp net/tcp/tcp.cpp net/tcp/connection.cpp net/tcp/connection_states.cpp net/tcp/write_queue.cpp net/tcp/rttm.cpp net/tcp/listener.cpp - net/tcp/read_buffer.cpp net/tcp/read_request.cpp + net/tcp/read_buffer.cpp net/tcp/read_request.cpp net/tcp/stream.cpp net/ip4/icmp4.cpp net/ip4/udp.cpp net/ip4/udp_socket.cpp net/ip6/ip6.cpp net/dns/dns.cpp net/dns/client.cpp net/dhcp/dh4client.cpp net/dhcp/dhcpd.cpp diff --git a/src/net/http/basic_client.cpp b/src/net/http/basic_client.cpp index 0131a32d0e..2b72ed5cae 100644 --- a/src/net/http/basic_client.cpp +++ b/src/net/http/basic_client.cpp @@ -290,7 +290,7 @@ namespace http { // no non-occupied connections, emplace a new one cset.push_back(std::make_unique( - *this, std::make_unique(tcp_.connect(host))) + *this, std::make_unique(tcp_.connect(host))) ); return *cset.back(); } diff --git a/src/net/http/client.cpp b/src/net/http/client.cpp index 1ee5278b1e..4a5a925e91 100644 --- a/src/net/http/client.cpp +++ b/src/net/http/client.cpp @@ -38,7 +38,7 @@ namespace http { return *conn; } - auto tcp_stream = std::make_unique(tcp_.connect(host)); + auto tcp_stream = std::make_unique(tcp_.connect(host)); auto tls_stream = std::make_unique(ssl_context, std::move(tcp_stream), true); cset.push_back(std::make_unique(*this, std::move(tls_stream))); diff --git a/src/net/https/botan_server.cpp b/src/net/https/botan_server.cpp index 217666da2c..2ed628bc6a 100644 --- a/src/net/https/botan_server.cpp +++ b/src/net/https/botan_server.cpp @@ -16,6 +16,7 @@ // limitations under the License. #include + #include #include #include @@ -74,7 +75,7 @@ namespace http { connect( std::make_unique ( - std::make_unique( + std::make_unique( std::move(conn)), rng, *credman) ); } diff --git a/src/net/https/openssl_server.cpp b/src/net/https/openssl_server.cpp index 6502d34c4f..c631d88dd9 100644 --- a/src/net/https/openssl_server.cpp +++ b/src/net/https/openssl_server.cpp @@ -36,7 +36,7 @@ namespace http void OpenSSL_server::on_connect(TCP_conn conn) { connect( - std::make_unique ((SSL_CTX*) m_ctx, std::make_unique(std::move(conn))) + std::make_unique ((SSL_CTX*) m_ctx, std::make_unique(std::move(conn))) ); } } // http diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 75cc17f3d0..e82fb1012a 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -25,22 +25,9 @@ #include #include -/*#include // remove me, sack debugging -std::ostream& operator<< (std::ostream& out, const net::tcp::sack::Entries& ent) { - for (auto el : ent) { - out << el << "\n"; - } - return out; -}*/ - - using namespace net::tcp; using namespace std; -int Connection::Stream::get_cpuid() const noexcept { - return m_tcp->host().get_cpuid(); -} - Connection::Connection(TCP& host, Socket local, Socket remote, ConnectCallback callback) : host_(host), local_(local), diff --git a/src/net/tcp/stream.cpp b/src/net/tcp/stream.cpp new file mode 100644 index 0000000000..51319df02f --- /dev/null +++ b/src/net/tcp/stream.cpp @@ -0,0 +1,18 @@ +#include +#include + +namespace net::tcp +{ + /*#include // remove me, sack debugging + std::ostream& operator<< (std::ostream& out, const net::tcp::sack::Entries& ent) { + for (auto el : ent) { + out << el << "\n"; + } + return out; + }*/ + + int Stream::get_cpuid() const noexcept { + return m_tcp->host().get_cpuid(); + } + +} From 9523de16587396d0578d2c794ae7403a6a9c2c8b Mon Sep 17 00:00:00 2001 From: Anders Schau Knatten Date: Tue, 22 May 2018 13:17:35 +0200 Subject: [PATCH 285/723] mremap: Be honest about not being implemented `mremap` used to return `SUCCESS` even though it's not implemented. This results in issues for users calling `realloc`. If we instead return `ENOSYS`, musl is able to detect that `mremap` isn't implemented and fall back to `malloc`/`memcpy`: https://git.musl-libc.org/cgit/musl/tree/src/malloc/malloc.c#n397 This was originally suggested by @fwsGonzo as a workaround for issues @andeplane ran into. After having a look at musl, I think it makes sense to do this officially too. Note: There are a bunch of other syscalls who also return SUCCESS even though they're stubbed out. Whether it makes sense to do the same to them has to be evaluated case by case taking the effect on the rest of the system into consideration. --- src/musl/mremap.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/musl/mremap.cpp b/src/musl/mremap.cpp index 6de60f361b..3941db8f1a 100644 --- a/src/musl/mremap.cpp +++ b/src/musl/mremap.cpp @@ -3,5 +3,5 @@ extern "C" long syscall_SYS_mremap() { STUB("mremap"); - return 0; + return -ENOSYS; } From 5e51637a03c4044dbf86e7377c807f77f0e72769 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 22 May 2018 16:25:05 +0200 Subject: [PATCH 286/723] net: Fix issues with TCP stream --- api/net/botan/tls_server.hpp | 2 +- api/net/openssl/tls_stream.hpp | 2 ++ api/net/stream.hpp | 5 ++--- api/net/tcp/stream.hpp | 11 +++++++++-- api/net/ws/websocket.hpp | 10 +++++----- src/net/ws/websocket.cpp | 28 ++++++++-------------------- 6 files changed, 27 insertions(+), 31 deletions(-) diff --git a/api/net/botan/tls_server.hpp b/api/net/botan/tls_server.hpp index 0255b5814b..8482f0ef92 100644 --- a/api/net/botan/tls_server.hpp +++ b/api/net/botan/tls_server.hpp @@ -174,7 +174,7 @@ class Server : public Botan::TLS::Callbacks, public net::Stream void tls_record_received(uint64_t, const uint8_t buf[], size_t buf_len) override { if (m_on_read) { - m_on_read(tcp::construct_buffer(buf, buf + buf_len)); + m_on_read(Stream::construct_buffer(buf, buf + buf_len)); } } diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index 23ace2510a..a40b6208ad 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -137,6 +137,8 @@ namespace openssl } inline TLS_stream::~TLS_stream() { + this->reset_callbacks(); + this->close(); SSL_free(this->m_ssl); } diff --git a/api/net/stream.hpp b/api/net/stream.hpp index 4ffcc59e11..9ae19d23bc 100644 --- a/api/net/stream.hpp +++ b/api/net/stream.hpp @@ -27,7 +27,6 @@ #include namespace net { - class Inet4; class Stream; using Stream_ptr = std::unique_ptr; /** @@ -38,9 +37,9 @@ namespace net { using buffer_t = std::shared_ptr>; using ptr = Stream_ptr; - /** Construct a shared vector used in TCP **/ + /** Construct a shared vector used by streams **/ template - buffer_t construct_buffer(Args&&... args) { + static buffer_t construct_buffer(Args&&... args) { return std::make_shared> (std::forward (args)...); } diff --git a/api/net/tcp/stream.hpp b/api/net/tcp/stream.hpp index 7276e9134d..a95c1b2172 100644 --- a/api/net/tcp/stream.hpp +++ b/api/net/tcp/stream.hpp @@ -21,11 +21,12 @@ namespace net::tcp { // stream for a nullptr makes no sense Expects(m_tcp != nullptr); + m_tcp->on_close({this, &Stream::close}); } ~Stream() { this->reset_callbacks(); - this->close(); + m_tcp->close(); } /** @@ -55,7 +56,9 @@ namespace net::tcp * @param[in] cb The close callback */ void on_close(CloseCallback cb) override - { m_tcp->on_close(cb); } + { + m_on_close = std::move(cb); + } /** * @brief Event for when data has been written. @@ -97,7 +100,10 @@ namespace net::tcp */ void close() override { + auto onclose = std::move(this->m_on_close); + m_tcp->reset_callbacks(); m_tcp->close(); + onclose(); } /** @@ -192,6 +198,7 @@ namespace net::tcp protected: Connection_ptr m_tcp; + CloseCallback m_on_close = nullptr; }; // < class Connection::Stream diff --git a/api/net/ws/websocket.hpp b/api/net/ws/websocket.hpp index 03a3852c56..b7db46a9d1 100644 --- a/api/net/ws/websocket.hpp +++ b/api/net/ws/websocket.hpp @@ -224,8 +224,9 @@ class WebSocket { //void ping(Stream::buffer_t, Timer::duration_t timeout); // close the websocket - void close(); - void close(uint16_t reason); + void close(uint16_t reason = 1000); + + void reset_callbacks(); // user callbacks close_func on_close = nullptr; @@ -285,10 +286,9 @@ class WebSocket { void read_data(Stream::buffer_t); bool write_opcode(op_code code, const char*, size_t); void failure(const std::string&); - void close_callback_once(uint16_t code); + void close_callback_once(); size_t create_message(const uint8_t*, size_t len); void finalize_message(); - void reset(); bool default_on_ping(const char*, size_t) { return true; } @@ -298,7 +298,7 @@ class WebSocket { if (on_pong_timeout) on_pong_timeout(*this); else - this->close(); + this->close(1000); } }; using WebSocket_ptr = WebSocket::WebSocket_ptr; diff --git a/src/net/ws/websocket.cpp b/src/net/ws/websocket.cpp index bf287f7e94..a78544123d 100644 --- a/src/net/ws/websocket.cpp +++ b/src/net/ws/websocket.cpp @@ -336,8 +336,7 @@ static Stream::buffer_t create_wsmsg(size_t len, op_code code, bool client) // create shared buffer with position at end of header auto buffer = tcp::construct_buffer(header_len); // create header on buffer - new (buffer->data()) ws_header; - auto& hdr = *(ws_header*) buffer->data(); + auto& hdr = *(new (buffer->data()) ws_header); hdr.bits = 0; hdr.set_final(); hdr.set_payload(len); @@ -421,18 +420,14 @@ WebSocket::WebSocket(net::Stream_ptr stream_ptr, bool client) { assert(stream != nullptr); this->stream->on_read(8*1024, {this, &WebSocket::read_data}); - this->stream->on_close({this, &WebSocket::close}); + this->stream->on_close({this, &WebSocket::close_callback_once}); } WebSocket::~WebSocket() { - if (stream != nullptr && stream->is_connected()) - this->close(1000); -} - -void WebSocket::close() -{ + this->reset_callbacks(); this->close(1000); } + void WebSocket::close(const uint16_t reason) { assert(stream != nullptr); @@ -441,16 +436,15 @@ void WebSocket::close(const uint16_t reason) this->write_opcode(op_code::CLOSE, "Closed", 6); /// close and unset socket this->stream->close(); - this->close_callback_once(reason); } -void WebSocket::close_callback_once(const uint16_t reason) +void WebSocket::close_callback_once() { auto close_func = std::move(this->on_close); - this->reset(); - if (close_func) close_func(reason); + this->reset_callbacks(); + if (close_func) close_func(1000); } -void WebSocket::reset() +void WebSocket::reset_callbacks() { this->on_close = nullptr; this->on_error = nullptr; @@ -458,12 +452,6 @@ void WebSocket::reset() this->on_ping = nullptr; this->on_pong = nullptr; this->on_pong_timeout = nullptr; - this->ping_timer.stop(); - if (this->stream != nullptr) - { - this->stream->reset_callbacks(); - this->stream = nullptr; - } } void WebSocket::failure(const std::string& reason) From da8822627f43724cfde1446c1d7651e21e5971fb Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 22 May 2018 16:36:11 +0200 Subject: [PATCH 287/723] net: Write WebSocket close reason properly --- src/net/ws/websocket.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/net/ws/websocket.cpp b/src/net/ws/websocket.cpp index 6c3b069a62..3bec59b662 100644 --- a/src/net/ws/websocket.cpp +++ b/src/net/ws/websocket.cpp @@ -432,8 +432,10 @@ void WebSocket::close(const uint16_t reason) { assert(stream != nullptr); /// send CLOSE message - if (this->stream->is_writable()) - this->write_opcode(op_code::CLOSE, "Closed", 6); + if (this->stream->is_writable()) { + uint16_t data = htons(reason); + this->write_opcode(op_code::CLOSE, (const char*) &data, sizeof(data)); + } /// close and unset socket this->stream->close(); } From 9d43b176946fdcb83c0d6231d0101cf3129dbe40 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 22 May 2018 17:17:41 +0200 Subject: [PATCH 288/723] test: Add stream.cpp to unittest sources --- api/net/stream.hpp | 4 +--- test/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/api/net/stream.hpp b/api/net/stream.hpp index 9ae19d23bc..cdb575785e 100644 --- a/api/net/stream.hpp +++ b/api/net/stream.hpp @@ -188,9 +188,7 @@ namespace net { virtual size_t serialize_to(void*) const = 0; - Stream() = default; - virtual ~Stream() {} - + virtual ~Stream() = default; }; // < class Stream } // < namespace net diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8e22b32b51..fdfafc04a7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -203,6 +203,7 @@ set(OS_SOURCES ${SRC}/net/tcp/tcp.cpp ${SRC}/net/tcp/read_buffer.cpp ${SRC}/net/tcp/read_request.cpp + ${SRC}/net/tcp/stream.cpp ${SRC}/net/tcp/write_queue.cpp ${SRC}/posix/fd.cpp ${SRC}/util/async.cpp From c45a3c5c7529f0c09f50cd8e6f800609cfd1d565 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 23 May 2018 09:36:04 +0200 Subject: [PATCH 289/723] net: Added extra bracers to ip6 init to silence warning --- api/net/ip6/addr.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index 38dbfe3dc4..468100e946 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -41,7 +41,7 @@ struct Invalid_Address : public std::runtime_error { */ struct Addr { Addr() - : i32{0, 0, 0, 0} {} + : i32{{0, 0, 0, 0}} {} Addr(uint16_t a1, uint16_t a2, uint16_t b1, uint16_t b2, uint16_t c1, uint16_t c2, uint16_t d1, uint16_t d2) From 5109479d34d2ad65f77096986355b2cb0d5920c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 23 May 2018 09:36:33 +0200 Subject: [PATCH 290/723] net: Remove unused(?) function close_callback_once in TLS stream --- api/net/openssl/tls_stream.hpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index a40b6208ad..2ccd20c93d 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -81,7 +81,6 @@ namespace openssl void tls_read(buffer_t); int tls_perform_stream_write(); int tls_perform_handshake(); - void close_callback_once(); bool handshake_completed() const noexcept; enum status_t { @@ -133,7 +132,7 @@ namespace openssl { // always-on callbacks m_transport->on_read(8192, {this, &TLS_stream::tls_read}); - m_transport->on_close({this, &TLS_stream::close_callback_once}); + m_transport->on_close({this, &TLS_stream::close}); } inline TLS_stream::~TLS_stream() { From ce23449d5d649fa2011d12993e18e18f928d4d4e Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 23 May 2018 09:56:45 +0200 Subject: [PATCH 291/723] x86: Remove template keyword from a few lines --- api/arch/x86/paging.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/arch/x86/paging.hpp b/api/arch/x86/paging.hpp index a3147fe5d9..073dbfa240 100644 --- a/api/arch/x86/paging.hpp +++ b/api/arch/x86/paging.hpp @@ -550,7 +550,7 @@ class Page_table { PG_PRINT(" Sub 0x%p want: %s\n", pdir, req.to_string().c_str()); - auto res = pdir->template map_r(req); + auto res = pdir->map_r(req); // We either get no result or a partial / correct one if (res) @@ -561,9 +561,9 @@ class Page_table { else { // If result is empty we had page size constraints that couldn't be met - auto sub_psize = pdir->template page_size; + auto sub_psize = pdir->page_size; Ensures((req.page_sizes & page_size) == 0 - or (pdir->template is_page_dir(req.lin) and (req.page_sizes & sub_psize)) + or (pdir->is_page_dir(req.lin) and (req.page_sizes & sub_psize)) or (sub_psize & req.page_sizes) == 0); } From b34290fa8932f344a479fe1d745fbe60ba1fd5af Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 23 May 2018 10:03:34 +0200 Subject: [PATCH 292/723] Test: New ip for kernel/liveupdate test --- test/README.md | 1 + test/kernel/integration/LiveUpdate/service.cpp | 2 +- test/kernel/integration/LiveUpdate/test.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/README.md b/test/README.md index 546cb50d8f..3cd1736b42 100644 --- a/test/README.md +++ b/test/README.md @@ -32,3 +32,4 @@ The following IP addresses are used by the services in the test folder. Make sur - stress: 10.0.0.42 - plugin/unik: 10.0.0.56 - kernel/modules: 10.0.0.53 +- kernel/liveupdate: 10.0.0.59 diff --git a/test/kernel/integration/LiveUpdate/service.cpp b/test/kernel/integration/LiveUpdate/service.cpp index 011d4f4ec1..bbc55528dc 100644 --- a/test/kernel/integration/LiveUpdate/service.cpp +++ b/test/kernel/integration/LiveUpdate/service.cpp @@ -34,7 +34,7 @@ void Service::start() if (OS::is_live_updated() == false) { auto& inet = net::Super_stack::get(0); - inet.network_config({10,0,0,49}, {255,255,255,0}, {10,0,0,1}); + inet.network_config({10,0,0,59}, {255,255,255,0}, {10,0,0,1}); setup_liveupdate_server(inet, 666, func); // signal test.py that the server is up diff --git a/test/kernel/integration/LiveUpdate/test.py b/test/kernel/integration/LiveUpdate/test.py index 8ea8e1b8a9..ab400f28e9 100755 --- a/test/kernel/integration/LiveUpdate/test.py +++ b/test/kernel/integration/LiveUpdate/test.py @@ -15,7 +15,7 @@ def begin_test(line): f = open('./service','rb') s = socket.socket() - s.connect(("10.0.0.49", 666)) + s.connect(("10.0.0.59", 666)) s.send(f.read()) s.close() From 59db5b064cb55d0355978188761f844337638ffa Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 23 May 2018 10:03:52 +0200 Subject: [PATCH 293/723] Test: New ip for kernel/term test --- test/README.md | 1 + test/kernel/integration/term/service.cpp | 2 +- test/kernel/integration/term/test.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/README.md b/test/README.md index 3cd1736b42..7b01ca6b53 100644 --- a/test/README.md +++ b/test/README.md @@ -33,3 +33,4 @@ The following IP addresses are used by the services in the test folder. Make sur - plugin/unik: 10.0.0.56 - kernel/modules: 10.0.0.53 - kernel/liveupdate: 10.0.0.59 +- kernel/term: 10.0.0.63 diff --git a/test/kernel/integration/term/service.cpp b/test/kernel/integration/term/service.cpp index 472aa2f21b..f7c7ea7498 100644 --- a/test/kernel/integration/term/service.cpp +++ b/test/kernel/integration/term/service.cpp @@ -24,7 +24,7 @@ void Service::start() { auto& inet = net::Super_stack::get(0); inet.network_config( - { 10,0,0,59 }, // IP + { 10,0,0,63 }, // IP { 255,255,255,0 }, // Netmask { 10,0,0,1 }); // GW diff --git a/test/kernel/integration/term/test.py b/test/kernel/integration/term/test.py index 9daffa0a69..286b5c1f63 100755 --- a/test/kernel/integration/term/test.py +++ b/test/kernel/integration/term/test.py @@ -13,7 +13,7 @@ def begin_test(line): s = socket.socket() - s.connect(("10.0.0.59", 23)) + s.connect(("10.0.0.63", 23)) s.send("netstat\r\n") result = s.recv(1024) print result From 1e3444f12317f84008c9e50b2bfc2349bf955614 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 23 May 2018 10:04:13 +0200 Subject: [PATCH 294/723] Test: New ip for posix/tcp test --- test/README.md | 1 + test/posix/integration/tcp/service.cpp | 2 +- test/posix/integration/tcp/test.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/README.md b/test/README.md index 7b01ca6b53..e7d44dd008 100644 --- a/test/README.md +++ b/test/README.md @@ -32,5 +32,6 @@ The following IP addresses are used by the services in the test folder. Make sur - stress: 10.0.0.42 - plugin/unik: 10.0.0.56 - kernel/modules: 10.0.0.53 +- posix/tcp: 10.0.0.57, 10.0.0.4 - kernel/liveupdate: 10.0.0.59 - kernel/term: 10.0.0.63 diff --git a/test/posix/integration/tcp/service.cpp b/test/posix/integration/tcp/service.cpp index efdafd7fe4..d19451ea7e 100644 --- a/test/posix/integration/tcp/service.cpp +++ b/test/posix/integration/tcp/service.cpp @@ -31,7 +31,7 @@ const uint16_t BUFSIZE = 2048; int main() { - auto&& inet = net::Inet::ifconfig({ 10, 0, 0, 51 }, // IP + auto&& inet = net::Inet::ifconfig({ 10, 0, 0, 57 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 4 }); // Gateway diff --git a/test/posix/integration/tcp/test.py b/test/posix/integration/tcp/test.py index 1c53a56f4e..7ce8aab968 100755 --- a/test/posix/integration/tcp/test.py +++ b/test/posix/integration/tcp/test.py @@ -37,7 +37,7 @@ def tear_down(): server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server.bind((S_HOST, S_PORT)) -HOST, PORT = '10.0.0.51', 1042 +HOST, PORT = '10.0.0.57', 1042 RECEIVED = '' From 9958befd078e2d941fe3baaa2ee7b7550de65102 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 23 May 2018 10:04:25 +0200 Subject: [PATCH 295/723] Test: New ip for posix/udp test --- test/README.md | 1 + test/posix/integration/udp/service.cpp | 2 +- test/posix/integration/udp/test.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/README.md b/test/README.md index e7d44dd008..ba57129ed8 100644 --- a/test/README.md +++ b/test/README.md @@ -33,5 +33,6 @@ The following IP addresses are used by the services in the test folder. Make sur - plugin/unik: 10.0.0.56 - kernel/modules: 10.0.0.53 - posix/tcp: 10.0.0.57, 10.0.0.4 +- posix/udp: 10.0.0.58, 10.0.0.3 - kernel/liveupdate: 10.0.0.59 - kernel/term: 10.0.0.63 diff --git a/test/posix/integration/udp/service.cpp b/test/posix/integration/udp/service.cpp index d08ec3ee79..3b5ad0a6cd 100644 --- a/test/posix/integration/udp/service.cpp +++ b/test/posix/integration/udp/service.cpp @@ -31,7 +31,7 @@ const uint16_t BUFSIZE = 2048; int main() { - auto&& inet = net::Inet::ifconfig({ 10, 0, 0, 50 }, // IP + auto&& inet = net::Inet::ifconfig({ 10, 0, 0, 58 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 3 }); // Gateway diff --git a/test/posix/integration/udp/test.py b/test/posix/integration/udp/test.py index dc019783c7..c41b341a8c 100755 --- a/test/posix/integration/udp/test.py +++ b/test/posix/integration/udp/test.py @@ -39,7 +39,7 @@ def tear_down(): server.bind((S_HOST, S_PORT)) -HOST, PORT = '10.0.0.50', 1042 +HOST, PORT = '10.0.0.58', 1042 RECEIVED = '' From 9a74bb7d2c0c4ff36a9599ae4aa1f23ef743d9a4 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 23 May 2018 10:04:46 +0200 Subject: [PATCH 296/723] Test: Updated ip for posix/syslog_plugin test --- test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/README.md b/test/README.md index ba57129ed8..382199cdf3 100644 --- a/test/README.md +++ b/test/README.md @@ -18,7 +18,6 @@ The following IP addresses are used by the services in the test folder. Make sur - http: 10.0.0.46 - icmp: 10.0.0.45 - icmp6: 10.0.0.52 -- ipv6: 10.0.0.47 - microLB: 10.0.0.1, 10.0.0.68, 10.0.0.69 - Is 10.0.0.1 a problem? - nat: 10.1.0.1, 192.1.0.1, 10.1.0.10, 192.1.0.192, 10.1.10.20 - router: Intrusive + time sensitive - runs alone @@ -32,6 +31,7 @@ The following IP addresses are used by the services in the test folder. Make sur - stress: 10.0.0.42 - plugin/unik: 10.0.0.56 - kernel/modules: 10.0.0.53 +- posix/syslog_plugin: 10.0.0.47, 10.0.0.2 - posix/tcp: 10.0.0.57, 10.0.0.4 - posix/udp: 10.0.0.58, 10.0.0.3 - kernel/liveupdate: 10.0.0.59 From 2daaccaf3be0df9985c0dfc53f552c47ca6dc628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 23 May 2018 10:17:01 +0200 Subject: [PATCH 297/723] examples: Add missing stream include in LiveUpdate --- examples/LiveUpdate/service.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/service.cpp index 44a1b8475b..aa88714512 100644 --- a/examples/LiveUpdate/service.cpp +++ b/examples/LiveUpdate/service.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "liu.hpp" static std::deque strims; static SSL_CTX* g_ctx = nullptr; From 83f55989a111e26a2c7958f4c01ceb1f38b33cf5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 23 May 2018 10:37:20 +0200 Subject: [PATCH 298/723] net: Use deferred closing on TLS stream --- api/net/openssl/tls_stream.hpp | 48 +++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index 2ccd20c93d..e59363794e 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -4,7 +4,7 @@ #include #include -#define VERBOSE_OPENSSL +//#define VERBOSE_OPENSSL #ifdef VERBOSE_OPENSSL #define TLS_PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else @@ -82,6 +82,7 @@ namespace openssl int tls_perform_stream_write(); int tls_perform_handshake(); bool handshake_completed() const noexcept; + void close_callback_once(); enum status_t { STATUS_OK, @@ -94,6 +95,8 @@ namespace openssl SSL* m_ssl = nullptr; BIO* m_bio_rd = nullptr; BIO* m_bio_wr = nullptr; + bool m_busy = false; + bool m_deferred_close = false; ConnectCallback m_on_connect = nullptr; ReadCallback m_on_read = nullptr; WriteCallback m_on_write = nullptr; @@ -119,7 +122,7 @@ namespace openssl SSL_set_bio(this->m_ssl, this->m_bio_rd, this->m_bio_wr); // always-on callbacks m_transport->on_read(8192, {this, &TLS_stream::tls_read}); - m_transport->on_close({this, &TLS_stream::close}); + m_transport->on_close({this, &TLS_stream::close_callback_once}); // start TLS handshake process if (outgoing == true) @@ -132,12 +135,13 @@ namespace openssl { // always-on callbacks m_transport->on_read(8192, {this, &TLS_stream::tls_read}); - m_transport->on_close({this, &TLS_stream::close}); + m_transport->on_close({this, &TLS_stream::close_callback_once}); } inline TLS_stream::~TLS_stream() { this->reset_callbacks(); - this->close(); + if (m_transport->is_connected()) + m_transport->close(); SSL_free(this->m_ssl); } @@ -219,8 +223,11 @@ namespace openssl n = SSL_read(this->m_ssl, temp, sizeof(temp)); if (n > 0) { auto buf = net::Stream::construct_buffer(temp, temp + n); - if (m_on_read) m_on_read(std::move(buf)); - + if (m_on_read) { + this->m_busy = true; + m_on_read(std::move(buf)); + this->m_busy = false; + } } } while (n > 0); // this goes here? @@ -228,6 +235,9 @@ namespace openssl TLS_PRINT("TLS_stream::SSL_read closed during read\n"); return; } + if (this->m_deferred_close) { + this->close(); return; + } auto status = this->status(n); // did peer request stream renegotiation? @@ -242,6 +252,11 @@ namespace openssl this->close(); return; } + // check deferred closing + if (this->m_deferred_close) { + this->close(); return; + } + } // while it < end } // tls_read() @@ -256,7 +271,11 @@ namespace openssl int n = BIO_read(this->m_bio_wr, buffer->data(), buffer->size()); assert(n == pending); m_transport->write(buffer); - if (m_on_write) m_on_write(n); + if (m_on_write) { + this->m_busy = true; + m_on_write(n); + this->m_busy = false; + } return n; } else { @@ -296,8 +315,18 @@ namespace openssl inline void TLS_stream::close() { ERR_clear_error(); - m_transport->close(); - CloseCallback func = std::move(m_on_close); + if (this->m_busy) { + m_deferred_close = true; return; + } + CloseCallback func = std::move(this->m_on_close); + this->reset_callbacks(); + if (m_transport->is_connected()) + m_transport->close(); + if (func) func(); + } + inline void TLS_stream::close_callback_once() + { + CloseCallback func = std::move(this->m_on_close); this->reset_callbacks(); if (func) func(); } @@ -312,7 +341,6 @@ namespace openssl this->m_on_connect = nullptr; this->m_on_read = nullptr; this->m_on_write = nullptr; - this->m_transport->reset_callbacks(); // why this? } inline bool TLS_stream::handshake_completed() const noexcept From 27e87926a9c32cd9c28957fee9a22fb4d723bdd6 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Wed, 23 May 2018 15:00:45 +0530 Subject: [PATCH 299/723] test: icmp6: Reduce the range to 9. Stop the vm after tests --- test/net/integration/icmp6/test.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/net/integration/icmp6/test.py b/test/net/integration/icmp6/test.py index 5728a5b98e..68300c8775 100755 --- a/test/net/integration/icmp6/test.py +++ b/test/net/integration/icmp6/test.py @@ -25,7 +25,7 @@ def start_icmp_test(trigger_line): print color.INFO(""), "Performing ping6 test" output_data = "" - for x in range(0, 11): + for x in range(0, 9): output_data += vm.readline() print output_data @@ -44,6 +44,13 @@ def start_icmp_test(trigger_line): else: print color.FAIL(""), "Ping test FAILED" + if num_successes == 1: + vm.exit(0, " All ICMP tests succeeded. Process returned 0 exit status") + else: + num_fails = 1 - num_successes + res = " " + str(num_fails) + " ICMP6 test(s) failed" + vm.exit(1, res) + vm.on_output("Service IPv4 address: 10.0.0.52, IPv6 address: fe80:0:0:0:e823:fcff:fef4:85bd", start_icmp_test); # Boot the VM, taking a timeout as parameter From 33cb907bb5b017ab83c15b7b9afb9a79d3280bbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 23 May 2018 13:03:43 +0200 Subject: [PATCH 300/723] tcp: Avoid reentrance when signaling close more than once --- api/net/tcp/connection.hpp | 2 ++ src/net/tcp/connection.cpp | 9 +++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index b182871445..b098fa390a 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -655,6 +655,8 @@ class Connection { /** Time Wait / DACK timeout timer */ Timer timewait_dack_timer; + bool close_signaled_ = false; + /** Number of retransmission attempts on the packet first in RT-queue */ int8_t rtx_attempt_ = 0; diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index e25591ec84..a98c2ce86f 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -1060,7 +1060,12 @@ void Connection::signal_connect(const bool success) (success) ? on_connect_(retrieve_shared()) : on_connect_(nullptr); } -void Connection::signal_close() { +void Connection::signal_close() +{ + if(UNLIKELY(close_signaled_)) + return; + close_signaled_ = true; + debug(" It's time to delete this connection. \n"); // call user callback @@ -1092,7 +1097,7 @@ void Connection::clean_up() { read_request->callback.reset(); _on_cleanup_.reset(); - debug2(" Succesfully cleaned up %s\n", to_string().c_str()); + debug(" Succesfully cleaned up %s\n", to_string().c_str()); } std::string Connection::TCB::to_string() const { From dacaa2c129155c1459a1e127e7fbc13dfec8313c Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 23 May 2018 10:50:06 +0200 Subject: [PATCH 301/723] Test: Made net/udp time_sensitive --- test/net/integration/udp/vm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/net/integration/udp/vm.json b/test/net/integration/udp/vm.json index cac6ea2460..f44841fea9 100644 --- a/test/net/integration/udp/vm.json +++ b/test/net/integration/udp/vm.json @@ -1,5 +1,6 @@ { "image" : "test_udp.img", "net" : [{"device" : "virtio", "mac" : "c0:01:0a:00:00:2a"}], - "mem" : 256 + "mem" : 256, + "time_sensitive": "True" } From 4413c61de806511f114d8453c7cd07ffb9ca5517 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 23 May 2018 13:15:38 +0200 Subject: [PATCH 302/723] Test: Hard dependency on websocket client version for test. --- test/net/integration/websocket/setup.sh | 2 +- test/net/integration/websocket/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/net/integration/websocket/setup.sh b/test/net/integration/websocket/setup.sh index 8c4dcc6173..946964a97d 100755 --- a/test/net/integration/websocket/setup.sh +++ b/test/net/integration/websocket/setup.sh @@ -1,3 +1,3 @@ #!/bin/bash set -e -sudo -H pip -q install ws4py +sudo -H pip -q install ws4py==0.4.2 diff --git a/test/net/integration/websocket/test.py b/test/net/integration/websocket/test.py index fd7c0d5897..a52ca91b77 100755 --- a/test/net/integration/websocket/test.py +++ b/test/net/integration/websocket/test.py @@ -21,9 +21,9 @@ def closed(self, code, reason=None): print " Closed down", code, reason def received_message(self, m): - print " received message" self.count += 1 if self.count >= 1000: + print " received ", self.count, "messages. Closing." self.close(reason='Bye bye') def startBenchmark(line): From cd7b392e0667526cbf0a9133fe5b06b62ec290e5 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 23 May 2018 14:09:57 +0200 Subject: [PATCH 303/723] Testrunner: Make sure num_cpus is actually an int. --- test/testrunner.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/testrunner.py b/test/testrunner.py index fbdd4548e3..2cc339c175 100755 --- a/test/testrunner.py +++ b/test/testrunner.py @@ -51,7 +51,7 @@ parser.add_argument("-j", "--junit-xml", dest="junit", action="store_true", help="Produce junit xml results") -parser.add_argument("-p", "--parallel-tests", dest="parallel", default=0, +parser.add_argument("-p", "--parallel-tests", dest="parallel", default=0, type=int, help="How many tests to run at once in parallell, \ overrides cpu count which is default") @@ -309,6 +309,7 @@ def integration_tests(tests): num_cpus = args.parallel else: num_cpus = multiprocessing.cpu_count() + num_cpus = int(num_cpus) # Collect test results print pretty.HEADER("Collecting integration test results, on {0} cpu(s)".format(num_cpus)) From 16396fef1bb5fc047cd5ff3e4699031a5b7b6cad Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 23 May 2018 15:53:01 +0200 Subject: [PATCH 304/723] Test: Added sleep to websocket test --- test/net/integration/websocket/test.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/net/integration/websocket/test.py b/test/net/integration/websocket/test.py index a52ca91b77..c2ed38df81 100755 --- a/test/net/integration/websocket/test.py +++ b/test/net/integration/websocket/test.py @@ -4,6 +4,7 @@ import sys import subprocess import thread +import time includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) @@ -16,10 +17,24 @@ class DummyClient(WebSocketClient): def opened(self): self.count = 0 print " Opened" + time.sleep(1) def closed(self, code, reason=None): print " Closed down", code, reason + def handshake_ok(self): + print " Handshake ok" + self._th.start() + + def close(self, code=1000, reason=''): + print "close is called, code: {0}, reason: {1}".format(code, reason) + + if not self.client_terminated: + self.client_terminated = True + + self._write(self.stream.close(code=code, reason=reason).single(mask=True)) + + def received_message(self, m): self.count += 1 if self.count >= 1000: @@ -32,7 +47,7 @@ def startBenchmark(line): ws = DummyClient('ws://10.0.0.54:8000/', protocols=['http-only', 'chat']) print " WS-client connecting" ws.connect() - print " WS-client conneted, doing run_forever" + print " WS-client connected, doing run_forever" ws.run_forever() print " Finished running forever" except KeyboardInterrupt: From 5eafffe154e8f1f3e989d270be2a042aecdc0db6 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 23 May 2018 15:56:26 +0200 Subject: [PATCH 305/723] Test: Increased timeout to 20 seconds for two vlan and nat tests --- test/net/integration/nat/test.py | 2 +- test/net/integration/vlan/test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/net/integration/nat/test.py b/test/net/integration/nat/test.py index 7b728094ea..dd06c8b6e4 100755 --- a/test/net/integration/nat/test.py +++ b/test/net/integration/nat/test.py @@ -8,4 +8,4 @@ sys.path.insert(0,includeos_src) from vmrunner import vmrunner -vmrunner.vms[0].cmake().boot(10).clean() +vmrunner.vms[0].cmake().boot(20).clean() diff --git a/test/net/integration/vlan/test.py b/test/net/integration/vlan/test.py index 7b728094ea..dd06c8b6e4 100755 --- a/test/net/integration/vlan/test.py +++ b/test/net/integration/vlan/test.py @@ -8,4 +8,4 @@ sys.path.insert(0,includeos_src) from vmrunner import vmrunner -vmrunner.vms[0].cmake().boot(10).clean() +vmrunner.vms[0].cmake().boot(20).clean() From bb51b2221c5cd10ae3016ae09fbfd710c276befc Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 23 May 2018 16:42:46 +0200 Subject: [PATCH 306/723] net: Add deferred close to Botan TLS stream --- api/net/botan/tls_server.hpp | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/api/net/botan/tls_server.hpp b/api/net/botan/tls_server.hpp index 8482f0ef92..721ea92656 100644 --- a/api/net/botan/tls_server.hpp +++ b/api/net/botan/tls_server.hpp @@ -74,7 +74,11 @@ class Server : public Botan::TLS::Callbacks, public net::Stream m_tls.send(buf->data(), buf->size()); } - void close() override { + void close() override + { + if (this->m_busy) { + this->m_deferred_close = true; return; + } m_transport->close(); CloseCallback cb = std::move(m_on_close); this->reset_callbacks(); @@ -168,13 +172,21 @@ class Server : public Botan::TLS::Callbacks, public net::Stream void tls_emit_data(const uint8_t buf[], size_t len) override { m_transport->write(buf, len); - if (m_on_write) m_on_write(len); + if (m_on_write) { + this->m_busy = true; + m_on_write(len); + this->m_busy = false; + if (this->m_deferred_close) this->close(); + } } void tls_record_received(uint64_t, const uint8_t buf[], size_t buf_len) override { if (m_on_read) { + this->m_busy = true; m_on_read(Stream::construct_buffer(buf, buf + buf_len)); + this->m_busy = false; + if (this->m_deferred_close) this->close(); } } @@ -188,6 +200,8 @@ class Server : public Botan::TLS::Callbacks, public net::Stream Stream::WriteCallback m_on_write = nullptr; Stream::ConnectCallback m_on_connect = nullptr; Stream::CloseCallback m_on_close = nullptr; + bool m_busy = false; + bool m_deferred_close = false; Botan::Credentials_Manager& m_creds; Botan::TLS::Strict_Policy m_policy; From 7fe463213725a6d42277879c3692d38056feeec9 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 23 May 2018 16:50:51 +0200 Subject: [PATCH 307/723] x86: Read UUID in endian-neutral way --- src/platform/x86_pc/smbios.cpp | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/platform/x86_pc/smbios.cpp b/src/platform/x86_pc/smbios.cpp index 8f4e39519c..2b92c44cd2 100644 --- a/src/platform/x86_pc/smbios.cpp +++ b/src/platform/x86_pc/smbios.cpp @@ -126,13 +126,10 @@ namespace x86 INFO2("Product name: %s", hdr->get_string(hdr->data[1])); { char uuid[33]; - snprintf(uuid, sizeof(uuid), - "%08x%08x%08x%08x", - *(uint32_t*) &hdr->data[4], - *(uint32_t*) &hdr->data[8], - *(uint32_t*) &hdr->data[12], - *(uint32_t*) &hdr->data[16]); - sysinfo.uuid = std::string(uuid); + for (int i = 0; i < 16; i++) { + sprintf(&uuid[i*2], "%02hhx", hdr->data[i]); + } + sysinfo.uuid = std::string(uuid, 32); INFO2("System UUID: %s", sysinfo.uuid.c_str()); } break; From f4f7f5b1f26120909b9f5e73e5d220d22c0e66c9 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 23 May 2018 17:23:20 +0200 Subject: [PATCH 308/723] net: Add deferred close to WebSocket --- api/net/ws/websocket.hpp | 6 ++++-- src/net/ws/websocket.cpp | 15 ++++++++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/api/net/ws/websocket.hpp b/api/net/ws/websocket.hpp index b7db46a9d1..1a737b5c4e 100644 --- a/api/net/ws/websocket.hpp +++ b/api/net/ws/websocket.hpp @@ -270,14 +270,16 @@ class WebSocket { static std::pair deserialize_from(const void*); WebSocket(net::Stream_ptr, bool); - ~WebSocket(); + ~WebSocket() = default; private: net::Stream_ptr stream; Timer ping_timer{{this, &WebSocket::pong_timeout}}; Message_ptr message; uint32_t max_msg_size; - bool clientside; + bool clientside; + bool m_busy = false; + uint16_t m_deferred_close = 0; WebSocket(const WebSocket&) = delete; WebSocket(WebSocket&&) = delete; diff --git a/src/net/ws/websocket.cpp b/src/net/ws/websocket.cpp index 3bec59b662..eba9ff5db5 100644 --- a/src/net/ws/websocket.cpp +++ b/src/net/ws/websocket.cpp @@ -295,8 +295,11 @@ void WebSocket::finalize_message() case op_code::TEXT: case op_code::BINARY: /// .. call on_read - if (on_read != nullptr) { - on_read(std::move(message)); + if (this->on_read) { + this->m_busy = true; + this->on_read(std::move(message)); + this->m_busy = false; + if (this->m_deferred_close) this->close(this->m_deferred_close); } return; case op_code::CLOSE: @@ -422,14 +425,12 @@ WebSocket::WebSocket(net::Stream_ptr stream_ptr, bool client) this->stream->on_read(8*1024, {this, &WebSocket::read_data}); this->stream->on_close({this, &WebSocket::close_callback_once}); } -WebSocket::~WebSocket() -{ - this->reset_callbacks(); - this->close(1000); -} void WebSocket::close(const uint16_t reason) { + if (this->m_busy) { + this->m_deferred_close = reason; return; + } assert(stream != nullptr); /// send CLOSE message if (this->stream->is_writable()) { From 007b0d917f8896874669cd1c52105c621309ad07 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 23 May 2018 17:26:42 +0200 Subject: [PATCH 309/723] net: Assert not busy on destructor of all complex Streams --- api/net/botan/tls_server.hpp | 3 +++ api/net/openssl/tls_stream.hpp | 4 +--- api/net/ws/websocket.hpp | 4 +++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/net/botan/tls_server.hpp b/api/net/botan/tls_server.hpp index 721ea92656..c2f46414f2 100644 --- a/api/net/botan/tls_server.hpp +++ b/api/net/botan/tls_server.hpp @@ -45,6 +45,9 @@ class Server : public Botan::TLS::Callbacks, public net::Stream // default read callback m_transport->on_read(4096, {this, &Server::tls_read}); } + ~Server() { + assert(m_busy == false && "Cannot delete stream while in its call stack"); + } void on_read(size_t bs, ReadCallback cb) override { diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index e59363794e..88222736f6 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -139,9 +139,7 @@ namespace openssl } inline TLS_stream::~TLS_stream() { - this->reset_callbacks(); - if (m_transport->is_connected()) - m_transport->close(); + assert(m_busy == false && "Cannot delete stream while in its call stack"); SSL_free(this->m_ssl); } diff --git a/api/net/ws/websocket.hpp b/api/net/ws/websocket.hpp index 1a737b5c4e..9e99f63ea6 100644 --- a/api/net/ws/websocket.hpp +++ b/api/net/ws/websocket.hpp @@ -270,7 +270,9 @@ class WebSocket { static std::pair deserialize_from(const void*); WebSocket(net::Stream_ptr, bool); - ~WebSocket() = default; + ~WebSocket() { + assert(m_busy == false && "Cannot delete stream while in its call stack"); + } private: net::Stream_ptr stream; From 17e228b5dc6a4250a7cb7e443c3a2962fcf980c9 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 23 May 2018 17:37:23 +0200 Subject: [PATCH 310/723] net: Fix close issue with Botan TLS stream --- api/net/botan/tls_server.hpp | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/api/net/botan/tls_server.hpp b/api/net/botan/tls_server.hpp index c2f46414f2..06db83cf7d 100644 --- a/api/net/botan/tls_server.hpp +++ b/api/net/botan/tls_server.hpp @@ -44,6 +44,7 @@ class Server : public Botan::TLS::Callbacks, public net::Stream assert(m_transport->is_connected()); // default read callback m_transport->on_read(4096, {this, &Server::tls_read}); + m_transport->on_close({this, &Server::close_callback_once}); } ~Server() { assert(m_busy == false && "Cannot delete stream while in its call stack"); @@ -82,7 +83,13 @@ class Server : public Botan::TLS::Callbacks, public net::Stream if (this->m_busy) { this->m_deferred_close = true; return; } + CloseCallback cb = std::move(m_on_close); + this->reset_callbacks(); m_transport->close(); + if (cb) cb(); + } + void close_callback_once() + { CloseCallback cb = std::move(m_on_close); this->reset_callbacks(); if (cb) cb(); @@ -97,7 +104,6 @@ class Server : public Botan::TLS::Callbacks, public net::Stream m_on_write = nullptr; m_on_connect = nullptr; m_on_close = nullptr; - m_transport->reset_callbacks(); } net::Socket local() const override { @@ -142,6 +148,7 @@ class Server : public Botan::TLS::Callbacks, public net::Stream protected: void tls_read(buffer_t buf) { + this->m_busy = true; try { int rem = m_tls.received_data(buf->data(), buf->size()); @@ -153,6 +160,8 @@ class Server : public Botan::TLS::Callbacks, public net::Stream printf("Fatal TLS error %s\n", e.what()); this->close(); } + this->m_busy = false; + if (this->m_deferred_close) this->close(); } void tls_alert(Botan::TLS::Alert alert) override @@ -186,10 +195,7 @@ class Server : public Botan::TLS::Callbacks, public net::Stream void tls_record_received(uint64_t, const uint8_t buf[], size_t buf_len) override { if (m_on_read) { - this->m_busy = true; m_on_read(Stream::construct_buffer(buf, buf + buf_len)); - this->m_busy = false; - if (this->m_deferred_close) this->close(); } } From 17bd8a7414a62cc3cda3728d48cda0905ceab06a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 23 May 2018 17:45:20 +0200 Subject: [PATCH 311/723] net: Remove abort() from stream API --- api/net/botan/tls_server.hpp | 4 ---- api/net/http/connection.hpp | 2 +- api/net/openssl/tls_stream.hpp | 6 ------ api/net/stream.hpp | 5 ----- api/net/tcp/stream.hpp | 6 ------ lib/microLB/micro_lb/balancer.cpp | 2 +- 6 files changed, 2 insertions(+), 23 deletions(-) diff --git a/api/net/botan/tls_server.hpp b/api/net/botan/tls_server.hpp index 06db83cf7d..b187243069 100644 --- a/api/net/botan/tls_server.hpp +++ b/api/net/botan/tls_server.hpp @@ -94,10 +94,6 @@ class Server : public Botan::TLS::Callbacks, public net::Stream this->reset_callbacks(); if (cb) cb(); } - void abort() override { - m_transport->abort(); - this->close(); - } void reset_callbacks() override { m_on_read = nullptr; diff --git a/api/net/http/connection.hpp b/api/net/http/connection.hpp index 3ab51cf72e..dc206eabc8 100644 --- a/api/net/http/connection.hpp +++ b/api/net/http/connection.hpp @@ -49,7 +49,7 @@ namespace http { { return peer_; } void timeout() - { stream_->is_closing() ? stream_->abort() : stream_->close(); } + { stream_->close(); } auto& stream() const { return stream_; } diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index 88222736f6..2f183e58c5 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -25,7 +25,6 @@ namespace openssl void write(const std::string&) override; void write(const void* buf, size_t n) override; void close() override; - void abort() override; void reset_callbacks() override; net::Socket local() const override { @@ -328,11 +327,6 @@ namespace openssl this->reset_callbacks(); if (func) func(); } - inline void TLS_stream::abort() - { - m_transport->abort(); - this->close(); - } inline void TLS_stream::reset_callbacks() { this->m_on_close = nullptr; diff --git a/api/net/stream.hpp b/api/net/stream.hpp index cdb575785e..ab471c10bf 100644 --- a/api/net/stream.hpp +++ b/api/net/stream.hpp @@ -108,11 +108,6 @@ namespace net { */ virtual void close() = 0; - /** - * @brief Aborts (terminates) the stream. - */ - virtual void abort() = 0; - /** * @brief Resets all callbacks. */ diff --git a/api/net/tcp/stream.hpp b/api/net/tcp/stream.hpp index a95c1b2172..0d068c5482 100644 --- a/api/net/tcp/stream.hpp +++ b/api/net/tcp/stream.hpp @@ -106,12 +106,6 @@ namespace net::tcp onclose(); } - /** - * @brief Aborts (terminates) the stream. - */ - void abort() override - { m_tcp->abort(); } - /** * @brief Resets all callbacks. */ diff --git a/lib/microLB/micro_lb/balancer.cpp b/lib/microLB/micro_lb/balancer.cpp index 099090a253..31410893ea 100644 --- a/lib/microLB/micro_lb/balancer.cpp +++ b/lib/microLB/micro_lb/balancer.cpp @@ -125,7 +125,7 @@ namespace microLB // prevent buffer bloat attack this->total += buf->size(); if (this->total > MAX_READQ_PER_NODE) { - conn->abort(); + conn->close(); } else { LBOUT("*** Queued %lu bytes\n", buf->size()); From c17bf2c1775bb392b278ec2cc23f247d1ffbd17a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 23 May 2018 17:45:39 +0200 Subject: [PATCH 312/723] x86: Remove template kw from paging summary --- api/arch/x86/paging.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/arch/x86/paging.hpp b/api/arch/x86/paging.hpp index f6f19f1aed..6b41660de0 100644 --- a/api/arch/x86/paging.hpp +++ b/api/arch/x86/paging.hpp @@ -319,7 +319,7 @@ class Page_table { printf("%.*s-+<%s> 0x%zx\n", print_lvl * 2, pad, util::Byte_r(page_size).to_string().c_str(), (void*)sub->start_addr()); } - sum += sub->template summary(print, print_lvl + 1); + sum += sub->summary(print, print_lvl + 1); } } if (print) { From 505d07a9f5f96593c9868485f420956f7454c50c Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 23 May 2018 17:48:11 +0200 Subject: [PATCH 313/723] x86: Remove optnone from IDT/__cpu_exception --- src/platform/x86_pc/idt.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/x86_pc/idt.cpp b/src/platform/x86_pc/idt.cpp index 38053a3602..996013c196 100644 --- a/src/platform/x86_pc/idt.cpp +++ b/src/platform/x86_pc/idt.cpp @@ -298,7 +298,7 @@ void __page_fault(uintptr_t* regs, uint32_t code) { } extern "C" -__attribute__((noreturn optnone, weak)) +__attribute__((noreturn, weak)) void __cpu_exception(uintptr_t* regs, int error, uint32_t code) { cpu_enable_panicking(); From 4af6eca8d62c7ae6e0f0c2fdd0b38586b2df281f Mon Sep 17 00:00:00 2001 From: Syeda Taiyeba Haroon Date: Thu, 24 May 2018 10:31:04 +0200 Subject: [PATCH 314/723] Net: Adding dhcpd test to time senstive. --- test/net/integration/dhcpd/vm.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/net/integration/dhcpd/vm.json b/test/net/integration/dhcpd/vm.json index 5f945654af..4b8dea8a20 100644 --- a/test/net/integration/dhcpd/vm.json +++ b/test/net/integration/dhcpd/vm.json @@ -4,5 +4,6 @@ {"device" : "virtio", "mac" : "c0:01:0a:00:00:2f"}, {"device" : "virtio", "mac" : "c0:01:0a:00:00:20"}, {"device" : "virtio", "mac" : "c0:01:0a:00:00:23"}], - "mem" : 128 + "mem" : 128, + "time_sensitive" : "True" } From d1253313191d91472162ad8b49e997e4c8b34a8f Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 24 May 2018 14:48:10 +0200 Subject: [PATCH 315/723] test: add INCLUDEOS_PREFIX/include to unittests for elf.h on mac --- test/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index eafe1c242e..c0e5cc3d04 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -2,6 +2,11 @@ cmake_minimum_required(VERSION 2.8.9) project(unittests C CXX) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +# IncludeOS install location +if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) + set(ENV{INCLUDEOS_PREFIX} /usr/local) +endif() + option(COVERAGE "Build with coverage generation" OFF) option(SILENT_BUILD "Build with some warnings turned off" ON) @@ -59,6 +64,7 @@ include_directories( ${INCLUDEOS_ROOT}/mod/GSL ${INCLUDEOS_ROOT}/mod/uzlib/src ${TEST}/lest/include + $ENV{INCLUDEOS_PREFIX}/include ) set(LEST_UTIL From 95459d98817bb7b2f26ecbd4e0fd00fe8be9485f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 24 May 2018 14:48:53 +0200 Subject: [PATCH 316/723] tcp: Implement clearing of sack blocks --- api/net/tcp/common.hpp | 1 + api/net/tcp/connection.hpp | 2 +- api/net/tcp/sack.hpp | 8 ++++++++ src/net/tcp/connection.cpp | 7 +++++-- test/net/unit/tcp_sack_test.cpp | 5 +++++ 5 files changed, 20 insertions(+), 3 deletions(-) diff --git a/api/net/tcp/common.hpp b/api/net/tcp/common.hpp index 80c30d3167..07233011af 100644 --- a/api/net/tcp/common.hpp +++ b/api/net/tcp/common.hpp @@ -37,6 +37,7 @@ namespace net { static constexpr bool default_timestamps {true}; // use of SACK static constexpr bool default_sack {true}; + static constexpr size_t default_sack_entries{32}; // maximum size of a TCP segment - later set based on MTU or peer static constexpr uint16_t default_mss {536}; // the maximum amount of half-open connections per port (listener) diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index b098fa390a..bca1e2db58 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -666,7 +666,7 @@ class Connection { /** State if connection is in TCP write queue or not. */ bool queued_; - using Sack_list = sack::List>; + using Sack_list = sack::List>; std::unique_ptr sack_list; /** If SACK is permitted (option has been seen from peer) */ bool sack_perm = false; diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index d851b1d483..00c50d3a92 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -126,6 +126,9 @@ class List { bool contains(const seq_t seq, const uint32_t len) const noexcept { return impl.contains(seq, len); } + void clear() noexcept + { impl.clear(); } + List_impl impl; }; @@ -295,6 +298,11 @@ class Fixed_list { return false; } + void clear() noexcept + { + blocks.clear(); + } + List blocks; }; diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index a98c2ce86f..06e6c2d2f9 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -71,8 +71,11 @@ void Connection::_on_read(size_t recv_bufsz, ReadCallback cb) read_request->callback = cb; // this will flush the current data to the user (if any) read_request->reset(recv_bufsz, seq_t(this->cb.RCV.NXT)); - // TODO: we should probably clear any SACK if it exists, - // due to reset() throwing away buffers + + // due to throwing away buffers (and all data) we also + // need to clear the sack list if anything is stored here. + if(sack_list) + sack_list->clear(); } } diff --git a/test/net/unit/tcp_sack_test.cpp b/test/net/unit/tcp_sack_test.cpp index f59e531f14..e168faf6a5 100644 --- a/test/net/unit/tcp_sack_test.cpp +++ b/test/net/unit/tcp_sack_test.cpp @@ -294,6 +294,11 @@ CASE("SACK block list is full") // Add a block that connects two blocks, which should free up one spot + // Clear list + sack_list.clear(); + EXPECT(sack_list.size() == 0); + EXPECT(sack_list.recent_entries() == expected({})); + } CASE("SACK Scoreboard - recv SACK") From 3b881dbe4901fe1e30cc19b33a76a47de707f7bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 24 May 2018 14:49:47 +0200 Subject: [PATCH 317/723] test: Include llvm5 libc++ when Clang --- test/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index eafe1c242e..8870a30d71 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -20,7 +20,7 @@ add_definitions(-DARCH_${ARCH}) add_definitions(-DARCH="${ARCH}") add_definitions(-DOS_VERSION="v0.0.0.1") -set(CMAKE_C_FLAGS "-g -O2 -std=c11 -Wall -Wextra") +set(CMAKE_C_FLAGS "-g -O0 -std=c11 -Wall -Wextra") set(NO_INFO "-DNO_INFO=1") if(INFO) @@ -32,15 +32,15 @@ if (DEBUG_INFO) set(NO_DEBUG "") endif() -set(CMAKE_CXX_FLAGS "-g -O2 -march=native -std=c++17 -Wall -Wextra -Wno-unused-function -D__id_t_defined -DUNITTESTS -DURI_THROW_ON_ERROR ${NO_INFO} ${NO_DEBUG} -DGSL_THROW_ON_CONTRACT_VIOLATION -Dlest_FEATURE_AUTO_REGISTER=1 -DHAVE_LEST_MAIN") +set(CMAKE_CXX_FLAGS "-g -O0 -march=native -std=c++17 -Wall -Wextra -Wno-unused-function -D__id_t_defined -DUNITTESTS -DURI_THROW_ON_ERROR ${NO_INFO} ${NO_DEBUG} -DGSL_THROW_ON_CONTRACT_VIOLATION -Dlest_FEATURE_AUTO_REGISTER=1 -DHAVE_LEST_MAIN") if (APPLE AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") message(STATUS "Including brew bundled libc++") execute_process(COMMAND brew "--prefix" "llvm@5" OUTPUT_VARIABLE BREW_LLVM) string(STRIP ${BREW_LLVM} BREW_LLVM) -set(BREW_LIBCXX_INC ${BREW_LLVM}/lib) +set(BREW_LIBCXX_INC "-L${BREW_LLVM}/lib -I${BREW_LLVM}/include/c++/v1") message(STATUS "Brew libc++ location: " ${BREW_LIBCXX_INC}) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -L${BREW_LIBCXX_INC} -Wno-unused-command-line-argument") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${BREW_LIBCXX_INC} -stdlib=libc++ -nostdinc++ -Wno-unused-command-line-argument") endif() if(NOT DEFINED ${INCLUDEOS_ROOT}) From 4bcc6adb7c595591178f274b698aa334c0dd7488 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Thu, 24 May 2018 14:50:30 +0200 Subject: [PATCH 318/723] test: add malloc.h for mac, avoid array deduction guide --- test/kernel/unit/memory.cpp | 4 ++-- test/lest_util/malloc.h | 6 ++++++ test/util/unit/lstack.cpp | 4 ---- 3 files changed, 8 insertions(+), 6 deletions(-) create mode 100644 test/lest_util/malloc.h diff --git a/test/kernel/unit/memory.cpp b/test/kernel/unit/memory.cpp index fb50fafeff..def0968bd7 100644 --- a/test/kernel/unit/memory.cpp +++ b/test/kernel/unit/memory.cpp @@ -319,7 +319,7 @@ SETUP ("Assuming a default page table setup") SECTION ("os::mem protect basics") { // Save some page sizes around the area we'll be mapping - std::array sizes_pre = {mem::active_page_size(5_GiB), + std::array sizes_pre {mem::active_page_size(5_GiB), mem::active_page_size(6_GiB), mem::active_page_size(6_GiB + 100_MiB), mem::active_page_size(7_GiB)}; @@ -363,7 +363,7 @@ SETUP ("Assuming a default page table setup") } // The rest of memory is largely as before except adjacent areas - std::array sizes_post = {mem::active_page_size(5_GiB), + std::array sizes_post {mem::active_page_size(5_GiB), mem::active_page_size(6_GiB), mem::active_page_size(prot_begin), mem::active_page_size(7_GiB)}; diff --git a/test/lest_util/malloc.h b/test/lest_util/malloc.h new file mode 100644 index 0000000000..7d56eab8ff --- /dev/null +++ b/test/lest_util/malloc.h @@ -0,0 +1,6 @@ +#pragma once +#ifdef __MACH__ +extern void* memalign(size_t alignment, size_t size); +#else +#include_next +#endif diff --git a/test/util/unit/lstack.cpp b/test/util/unit/lstack.cpp index 7b44da950b..06ca829786 100644 --- a/test/util/unit/lstack.cpp +++ b/test/util/unit/lstack.cpp @@ -16,11 +16,7 @@ #include #include -#ifdef __MACH__ -extern void* memalign(size_t alignment, size_t size); -#else #include -#endif using Lstack = util::alloc::Lstack<>; using Chunk = Lstack::Chunk; From 13f68ec63b49b01ebcce09053d14b25b76dffba3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Thu, 24 May 2018 15:51:35 +0200 Subject: [PATCH 319/723] test: GRUB test now uses xorriso/grub-mkrescue --- test/kernel/integration/grub/.gitignore | 1 + test/kernel/integration/grub/README.md | 10 ++++------ test/kernel/integration/grub/grub.cfg | 7 +++++++ test/kernel/integration/grub/grubiso.sh | 12 ++++++++++++ test/kernel/integration/grub/test.py | 4 ++-- test/kernel/integration/grub/vm.json | 2 +- 6 files changed, 27 insertions(+), 9 deletions(-) create mode 100644 test/kernel/integration/grub/.gitignore create mode 100644 test/kernel/integration/grub/grub.cfg create mode 100755 test/kernel/integration/grub/grubiso.sh diff --git a/test/kernel/integration/grub/.gitignore b/test/kernel/integration/grub/.gitignore new file mode 100644 index 0000000000..cf6e840170 --- /dev/null +++ b/test/kernel/integration/grub/.gitignore @@ -0,0 +1 @@ +temp_disk/ diff --git a/test/kernel/integration/grub/README.md b/test/kernel/integration/grub/README.md index 11ce4b3b22..bf13163c9d 100644 --- a/test/kernel/integration/grub/README.md +++ b/test/kernel/integration/grub/README.md @@ -1,9 +1,7 @@ # Test booting with GRUB -The test will create a disk image with a GRUB bootloader and a minimal IncludeOS service. The test only verifies that the service boots with GRUB andstarts successfully - nothing else. +The test will create a disk image with a GRUB bootloader and a minimal IncludeOS service. The test only verifies that the service boots with GRUB and starts successfully. -NOTE: The script that creates the image will: -* Require sudo -* Try to install GRUB 2.0 (using apt so very ubuntu specific) -* Mount stuff in a created local folder -* Use a loopback device +NOTE: +- Requires xorriso package +- Requires grub-pc package diff --git a/test/kernel/integration/grub/grub.cfg b/test/kernel/integration/grub/grub.cfg new file mode 100644 index 0000000000..98c0b9b2b9 --- /dev/null +++ b/test/kernel/integration/grub/grub.cfg @@ -0,0 +1,7 @@ +set timeout=0 +set default=0 # Set the default menu entry + +menuentry "IncludeOS" { + multiboot /boot/service + boot +} diff --git a/test/kernel/integration/grub/grubiso.sh b/test/kernel/integration/grub/grubiso.sh new file mode 100755 index 0000000000..e1e7135521 --- /dev/null +++ b/test/kernel/integration/grub/grubiso.sh @@ -0,0 +1,12 @@ +#!/bin/bash +LOCAL_DISK=temp_disk +SERVICE=$1 +GRUBIMG=build/grub.iso +set -e + +echo "Building $GRUBIMG..." +# create grub.iso +mkdir -p $LOCAL_DISK/boot/grub +cp $SERVICE $LOCAL_DISK/boot/service +cp grub.cfg $LOCAL_DISK/boot/grub +grub-mkrescue -d /usr/lib/grub/i386-pc -o $GRUBIMG $LOCAL_DISK diff --git a/test/kernel/integration/grub/test.py b/test/kernel/integration/grub/test.py index befd7487ff..bb4128798b 100755 --- a/test/kernel/integration/grub/test.py +++ b/test/kernel/integration/grub/test.py @@ -19,10 +19,10 @@ os.chdir("..") # Use grubify-script -grubify = includeos_src + "/etc/scripts/grubify.sh" +grubify = "grubiso.sh" # Create the GRUB image -subprocess.check_call(["bash",grubify,"build/test_grub", "-c"]) +subprocess.check_call(["bash",grubify,"build/test_grub"]) # Boot the image vm.boot(multiboot = False) diff --git a/test/kernel/integration/grub/vm.json b/test/kernel/integration/grub/vm.json index f63ab21228..dd58f961a1 100644 --- a/test/kernel/integration/grub/vm.json +++ b/test/kernel/integration/grub/vm.json @@ -1 +1 @@ -{"image" : "build/test_grub.grub.img" } +{"image" : "build/grub.iso" } From cc755e5be9344654541455ef1e1315e0b2c2d2f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 24 May 2018 16:30:41 +0200 Subject: [PATCH 320/723] microlb: Avoid close loop --- lib/microLB/micro_lb/balancer.cpp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/microLB/micro_lb/balancer.cpp b/lib/microLB/micro_lb/balancer.cpp index 31410893ea..5236cdf9d5 100644 --- a/lib/microLB/micro_lb/balancer.cpp +++ b/lib/microLB/micro_lb/balancer.cpp @@ -387,7 +387,7 @@ namespace microLB incoming->on_close( [&nodes = n, idx] () { nodes.get_session(idx).outgoing->close(); - nodes.get_session(idx).incoming->close(); + //nodes.get_session(idx).incoming->close(); }); outgoing->on_read(READQ_FOR_NODES, [this] (auto buf) { @@ -397,13 +397,9 @@ namespace microLB }); outgoing->on_close( [&nodes = n, idx] () { - nodes.get_session(idx).outgoing->close(); + //nodes.get_session(idx).outgoing->close(); nodes.get_session(idx).incoming->close(); }); - outgoing->on_close( - [&nodes = n, idx] () { - nodes.close_session(idx); - }); } bool Session::is_alive() const { return incoming != nullptr; From df6d307a5b24f341491612c94fb94ea320ea836c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 24 May 2018 16:31:41 +0200 Subject: [PATCH 321/723] test: Some additional glue when building tests with GCC on Mac --- test/CMakeLists.txt | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 08f7e8f126..6e7099c504 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -39,13 +39,18 @@ endif() set(CMAKE_CXX_FLAGS "-g -O0 -march=native -std=c++17 -Wall -Wextra -Wno-unused-function -D__id_t_defined -DUNITTESTS -DURI_THROW_ON_ERROR ${NO_INFO} ${NO_DEBUG} -DGSL_THROW_ON_CONTRACT_VIOLATION -Dlest_FEATURE_AUTO_REGISTER=1 -DHAVE_LEST_MAIN") -if (APPLE AND "${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") -message(STATUS "Including brew bundled libc++") -execute_process(COMMAND brew "--prefix" "llvm@5" OUTPUT_VARIABLE BREW_LLVM) -string(STRIP ${BREW_LLVM} BREW_LLVM) -set(BREW_LIBCXX_INC "-L${BREW_LLVM}/lib -I${BREW_LLVM}/include/c++/v1") -message(STATUS "Brew libc++ location: " ${BREW_LIBCXX_INC}) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${BREW_LIBCXX_INC} -stdlib=libc++ -nostdinc++ -Wno-unused-command-line-argument") +if (APPLE) + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + message(STATUS "Including brew bundled libc++") + execute_process(COMMAND brew "--prefix" "llvm@5" OUTPUT_VARIABLE BREW_LLVM) + string(STRIP ${BREW_LLVM} BREW_LLVM) + set(BREW_LIBCXX_INC "-L${BREW_LLVM}/lib -I${BREW_LLVM}/include/c++/v1") + message(STATUS "Brew libc++ location: " ${BREW_LIBCXX_INC}) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${BREW_LIBCXX_INC} -stdlib=libc++ -nostdinc++ -Wno-unused-command-line-argument") + else() + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mmacosx-version-min=10.12") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mmacosx-version-min=10.12") + endif() endif() if(NOT DEFINED ${INCLUDEOS_ROOT}) From 54e140500a0785b98d2d4580ad0954ccada94c90 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Thu, 24 May 2018 09:43:15 +0200 Subject: [PATCH 322/723] Test: Suppress output from apt command in router --- test/net/integration/router/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net/integration/router/setup.sh b/test/net/integration/router/setup.sh index dbeb0ebade..b30af4f081 100755 --- a/test/net/integration/router/setup.sh +++ b/test/net/integration/router/setup.sh @@ -14,7 +14,7 @@ alias server1="sudo ip netns exec $NSNAME" setup() { # TODO: it's probably not nice to install test deps here - sudo apt install -y iperf3 + sudo apt -qqq install -y iperf3 # Make sure the default bridge exists $INCLUDEOS_PREFIX/includeos/scripts/create_bridge.sh From 19118562fd04d9d53e62370d0a538acc34669022 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Fri, 25 May 2018 09:09:14 +0200 Subject: [PATCH 323/723] Testrunner: Added time duration print to output --- test/testrunner.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/testrunner.py b/test/testrunner.py index 2cc339c175..5583740fb1 100755 --- a/test/testrunner.py +++ b/test/testrunner.py @@ -74,6 +74,7 @@ def __init__(self, path, clean=False, command=['python', 'test.py'], name=None): self.path_ = path self.output_ = [] self.clean = clean + self.start_time = None self.properties_ = {"time_sensitive": False, "intrusive": False} # Extract category and type from the path variable # Category is linked to the top level folder e.g. net, fs, hw @@ -134,6 +135,7 @@ def __str__(self): ).format(x=self.__dict__) def start(self): + self.start_time = time.time() os.chdir(startdir + "/" + self.path_) if self.clean: self.clean_test() @@ -161,7 +163,11 @@ def clean_test(self): def print_start(self): - print "* {0:66} ".format(self.name_), + print "* {0:59} ".format(self.name_), + sys.stdout.flush() + + def print_duration(self): + print "{0:5.0f}s".format(time.time() - self.start_time), sys.stdout.flush() def wait_status(self): @@ -170,6 +176,7 @@ def wait_status(self): # Start and wait for the process self.proc_.communicate() + self.print_duration() with codecs.open('{}/log_stdout.log'.format(self.path_), encoding='utf-8', errors='replace') as log_stdout: self.output_.append(log_stdout.read()) From b55740b0d9a30f056dc47e382af782e04dcb28d9 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 25 May 2018 13:38:59 +0200 Subject: [PATCH 324/723] net: Add chain_length() to Packet --- api/net/packet.hpp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/api/net/packet.hpp b/api/net/packet.hpp index ca7fa794e2..a4b3639177 100644 --- a/api/net/packet.hpp +++ b/api/net/packet.hpp @@ -127,6 +127,16 @@ namespace net } } + int chain_length() const noexcept { + int count = 1; + auto* p = this; + while (p->chain_) { + p = p->chain_.get(); + count++; + } + return count; + } + /* Get the last packet in the chain */ Packet* last_in_chain() noexcept { return last_; } From fa29aae8a34656df1751b423dc7e5e0e2d253f18 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 25 May 2018 13:39:11 +0200 Subject: [PATCH 325/723] e1000: Increment sendq counter by packet chain length --- src/drivers/e1000.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drivers/e1000.cpp b/src/drivers/e1000.cpp index d70aec89f0..bb60a0ff72 100644 --- a/src/drivers/e1000.cpp +++ b/src/drivers/e1000.cpp @@ -587,11 +587,11 @@ uint16_t e1000::free_transmit_descr() const noexcept void e1000::transmit(net::Packet_ptr pckt) { if (pckt != nullptr) { + sendq_size += pckt->chain_length(); if (sendq == nullptr) sendq = std::move(pckt); else sendq->chain(std::move(pckt)); - sendq_size += 1; } // send as much as possible from sendq From 2d8cf1a8c58963ba2773f83e32363adbe2d796cb Mon Sep 17 00:00:00 2001 From: mhk Date: Mon, 28 May 2018 11:30:45 +0200 Subject: [PATCH 326/723] Fix compile error when using test_ukvm.sh Moved `extern "C"` to correct location. Otherwise, when executing `./test_ukvm.sh` the following compile error is generated: ``` includeos/src/service_name.cpp:24:8: error: expected unqualified-id before string constant extern "C" ^ ``` --- src/service_name.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/service_name.cpp b/src/service_name.cpp index a21e21a938..8e3b6e39f8 100644 --- a/src/service_name.cpp +++ b/src/service_name.cpp @@ -20,8 +20,8 @@ extern "C" __attribute__((noreturn)) void panic(const char* reason); -__attribute__((noreturn)) -extern "C" void abort(){ +extern "C" __attribute__((noreturn)) +void abort(){ panic("Abort called"); } From 234cf41b54f719681291efdd10d01c51dd8e596c Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 28 May 2018 15:01:38 +0200 Subject: [PATCH 327/723] system_log: Move systemlog phys to high memory --- api/kernel/memory.hpp | 12 ++++++++++++ src/plugins/system_log.cpp | 14 ++++++++++++++ src/service_name.cpp | 4 ++-- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/api/kernel/memory.hpp b/api/kernel/memory.hpp index 4c9f64fc49..dc08190963 100644 --- a/api/kernel/memory.hpp +++ b/api/kernel/memory.hpp @@ -147,6 +147,8 @@ namespace mem { /** Get the physical address to which linear address is mapped **/ uintptr_t virt_to_phys(uintptr_t linear); + void virtual_move(uintptr_t src, size_t size, uintptr_t dst, const char* label); + }} // os::mem @@ -302,6 +304,16 @@ namespace mem { return active_page_size((uintptr_t) addr); } + inline void + virtual_move(uintptr_t src, size_t size, uintptr_t dst, const char* label) + { + using namespace util::bitops; + const auto flags = os::mem::Access::read | os::mem::Access::write; + // setup @dst as new virt area for @src + os::mem::map({dst, src, flags, size}, label); + // unpresent @src + os::mem::protect(src, size, os::mem::Access::none); + } }} #endif diff --git a/src/plugins/system_log.cpp b/src/plugins/system_log.cpp index 1f86f909e4..9a17d71038 100644 --- a/src/plugins/system_log.cpp +++ b/src/plugins/system_log.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -27,7 +28,11 @@ static inline RingBuffer* get_mrb() inline char* get_ringbuffer_loc() { +#ifdef ARCH_x86_64 + return (char*) ((1ull << 46) - MRB_LOG_SIZE); +#else return (char*) OS::liveupdate_storage_area() - MRB_LOG_SIZE; +#endif } inline char* get_system_log_loc() @@ -74,6 +79,15 @@ void SystemLog::initialize() { INFO("SystemLog", "Initializing System Log"); +#ifdef ARCH_x86_64 + using namespace util::bitops; + const size_t size = MRB_LOG_SIZE - 4096; + const uintptr_t syslog_area = (uintptr_t) get_ringbuffer_loc() - 4096; + const uintptr_t origin_area = (uintptr_t) OS::liveupdate_storage_area() - size; + // move systemlog to high memory and unpresent physical + os::mem::virtual_move(origin_area, size, syslog_area, "SystemLog"); +#endif + auto& buffer = get_log_buffer(); mrb = buffer.get_mrb(); diff --git a/src/service_name.cpp b/src/service_name.cpp index a21e21a938..8e3b6e39f8 100644 --- a/src/service_name.cpp +++ b/src/service_name.cpp @@ -20,8 +20,8 @@ extern "C" __attribute__((noreturn)) void panic(const char* reason); -__attribute__((noreturn)) -extern "C" void abort(){ +extern "C" __attribute__((noreturn)) +void abort(){ panic("Abort called"); } From 3b8dd9ea47e7be98200e7ee840145828f581bbd9 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 28 May 2018 15:13:10 +0200 Subject: [PATCH 328/723] kernel: Silence kernel_start and some syscalls --- src/arch/x86_64/syscall_entry.cpp | 24 +++++------ src/platform/x86_pc/kernel_start.cpp | 60 ++++++++++++++++------------ src/platform/x86_pc/platform.cpp | 1 - 3 files changed, 46 insertions(+), 39 deletions(-) diff --git a/src/arch/x86_64/syscall_entry.cpp b/src/arch/x86_64/syscall_entry.cpp index edf7b0b031..e8374b7fbc 100644 --- a/src/arch/x86_64/syscall_entry.cpp +++ b/src/arch/x86_64/syscall_entry.cpp @@ -24,24 +24,24 @@ #ifdef __x86_64__ -int __arch_prctl(int code, uintptr_t ptr){ - +static int sys_prctl(int code, uintptr_t ptr) +{ switch(code){ case ARCH_SET_GS: - kprintf(" set_gs to %#lx\n", ptr); - if (!ptr) return -1; + //kprintf(" set_gs to %#lx\n", ptr); + if (UNLIKELY(!ptr)) return -EINVAL; x86::CPU::set_gs((void*)ptr); break; case ARCH_SET_FS: - kprintf(" set_fs to %#lx\n", ptr); - if (!ptr) return -1; + //kprintf(" set_fs to %#lx\n", ptr); + if (UNLIKELY(!ptr)) return -EINVAL; x86::CPU::set_fs((void*)ptr); break; case ARCH_GET_GS: - panic(" get gs \n"); + panic(" GET_GS called!\n"); break; case ARCH_GET_FS: - panic(" get gs \n"); + panic(" GET_FS called!\n"); break; } return 0; @@ -52,13 +52,13 @@ extern "C" uintptr_t syscall_entry(uint64_t n, uint64_t a1, uint64_t a2, uint64_t a3, uint64_t a4, uint64_t a5) { - kprintf(" no %lu (a1=%#lx a2=%#lx a3=%#lx a4=%#lx a5=%#lx) \n", - n, a1, a2, a3, a4, a5); - switch(n) { case 158: // arch_prctl - __arch_prctl(a1, a2); + sys_prctl(a1, a2); break; + default: + kprintf(" no %lu (a1=%#lx a2=%#lx a3=%#lx a4=%#lx a5=%#lx) \n", + n, a1, a2, a3, a4, a5); } return 0; } diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index c69d94c02c..377b032831 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -30,6 +30,13 @@ #undef Expects #define Expects(X) if (!(X)) { kprint("Expect failed: " #X "\n"); asm("cli;hlt"); } +//#define KERN_DEBUG 1 +#ifdef KERN_DEBUG +#define PRATTLE(fmt, ...) kprintf(fmt, ##__VA_ARGS__) +#else +#define PRATTLE(fmt, ...) /* fmt */ +#endif + extern "C" { void __init_serial1(); void __init_sanity_checks(); @@ -68,7 +75,7 @@ static void global_ctor_test(){ } int kernel_main(int, char * *, char * *) { - kprintf(" libc initialization complete \n"); + PRATTLE(" libc initialization complete \n"); Expects(__global_ctors_ok == 42); __libc_initialized = true; @@ -76,11 +83,11 @@ int kernel_main(int, char * *, char * *) { Elf_binary elf{{(char*)&_ELF_START_, &_ELF_END_ - &_ELF_START_}}; Expects(elf.is_ELF() && "ELF header intact"); - kprintf(" OS start \n"); + PRATTLE(" OS start \n"); // Initialize early OS, platform and devices OS::start(__grub_magic,__grub_addr); - kprintf(" post start \n"); + PRATTLE(" post start \n"); // Initialize common subsystems and call Service::start OS::post_start(); @@ -113,14 +120,13 @@ void kernel_start(uint32_t magic, uint32_t addr) __grub_magic = magic; __grub_addr = addr; - // TODO: remove extra verbosity after musl port stabilizes - kprintf("\n////////////////// IncludeOS kernel start ////////////////// \n"); - kprintf("* Booted with magic 0x%x, grub @ 0x%x \n* Init sanity\n", + PRATTLE("\n////////////////// IncludeOS kernel start ////////////////// \n"); + PRATTLE("* Booted with magic 0x%x, grub @ 0x%x \n* Init sanity\n", magic, addr); // generate checksums of read-only areas etc. __init_sanity_checks(); - kprintf("* Grub magic: 0x%x, grub info @ 0x%x\n", + PRATTLE("* Grub magic: 0x%x, grub info @ 0x%x\n", __grub_magic, __grub_addr); // Determine where free memory starts @@ -132,53 +138,55 @@ void kernel_start(uint32_t magic, uint32_t addr) free_mem_begin = _multiboot_free_begin(addr); memory_end = _multiboot_memory_end(addr); } - kprintf("* Free mem begin: 0x%zx, memory end: 0x%zx \n", + PRATTLE("* Free mem begin: 0x%zx, memory end: 0x%zx \n", free_mem_begin, memory_end); - kprintf("* Moving symbols. \n"); + PRATTLE("* Moving symbols. \n"); // Preserve symbols from the ELF binary free_mem_begin += _move_symbols(free_mem_begin); - kprintf("* Free mem moved to: %p \n", (void*) free_mem_begin); + PRATTLE("* Free mem moved to: %p \n", (void*) free_mem_begin); - kprintf("* Grub magic: 0x%x, grub info @ 0x%x\n", + PRATTLE("* Grub magic: 0x%x, grub info @ 0x%x\n", __grub_magic, __grub_addr); - kprintf("* Init .bss\n"); + PRATTLE("* Init .bss\n"); _init_bss(); - kprintf("* Init heap\n"); + PRATTLE("* Init heap\n"); OS::init_heap(free_mem_begin, memory_end); - kprintf("* Init syscalls\n"); + PRATTLE("* Init syscalls\n"); _init_syscalls(); - kprintf("* Init CPU exceptions\n"); + PRATTLE("* Init CPU exceptions\n"); x86::idt_initialize_for_cpu(0); - kprintf("* Init ELF parser\n"); + PRATTLE("* Init ELF parser\n"); _init_elf_parser(); - kprintf("* Thread local1: %i\n", __tl1__); + PRATTLE("* Thread local1: %i\n", __tl1__); - kprintf("* Elf start: %p\n", &_ELF_START_); + PRATTLE("* Elf start: %p\n", &_ELF_START_); auto* ehdr = (Elf64_Ehdr*)&_ELF_START_; auto* phdr = (Elf64_Phdr*)((char*)ehdr + ehdr->e_phoff); Expects(phdr); Elf_binary elf{{(char*)&_ELF_START_, &_ELF_END_ - &_ELF_START_}}; Expects(elf.is_ELF()); - size_t size = &_ELF_END_ - &_ELF_START_; Expects(phdr[0].p_type == PT_LOAD); - printf("* Elf ident: %s, program headers: %p\n", ehdr->e_ident, ehdr); - kprintf("\tElf size: %zu \n", size); +#ifdef KERN_DEBUG + PRATTLE("* Elf ident: %s, program headers: %p\n", ehdr->e_ident, ehdr); + size_t size = &_ELF_END_ - &_ELF_START_; + PRATTLE("\tElf size: %zu \n", size); for (int i = 0; i < ehdr->e_phnum; i++) { - kprintf("\tPhdr %i @ %p, va_addr: 0x%lx \n", i, &phdr[i], phdr[i].p_vaddr); + PRATTLE("\tPhdr %i @ %p, va_addr: 0x%lx \n", i, &phdr[i], phdr[i].p_vaddr); } +#endif // Build AUX-vector for C-runtime auxv_t aux[38]; - kprintf("* Initializing aux-vector @ %p\n", aux); + PRATTLE("* Initializing aux-vector @ %p\n", aux); int i = 0; aux[i++].set_long(AT_PAGESZ, 4096); @@ -220,18 +228,18 @@ void kernel_start(uint32_t magic, uint32_t addr) memcpy(&argv[6], aux, sizeof(auxv_t) * 38); #if defined(__x86_64__) - kprintf("* Initialize syscall MSR\n"); + PRATTLE("* Initialize syscall MSR (64-bit)\n"); uint64_t star_kernel_cs = 8ull << 32; uint64_t star_user_cs = 8ull << 48; uint64_t star = star_kernel_cs | star_user_cs; x86::CPU::write_msr(IA32_STAR, star); x86::CPU::write_msr(IA32_LSTAR, (uintptr_t)&__syscall_entry); #elif defined(__i386__) - kprintf("Initialize syscall MSR (32)\n"); + PRATTLE("Initialize syscall intr (32-bit)\n"); #warning Classical syscall interface missing for 32-bit #endif // GDB_ENTRY; - kprintf("* Starting libc initialization\n"); + PRATTLE("* Starting libc initialization\n"); __libc_start_main(kernel_main, argc, argv.data()); } diff --git a/src/platform/x86_pc/platform.cpp b/src/platform/x86_pc/platform.cpp index f67e2f117c..880a429f41 100644 --- a/src/platform/x86_pc/platform.cpp +++ b/src/platform/x86_pc/platform.cpp @@ -101,7 +101,6 @@ void __platform_init() // Print registered devices hw::Devices::print_devices(); - kprintf("Platform init done \n"); } #ifdef ARCH_i686 From a05a30842bb97696f52bcd229db8a8836462c0c8 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 28 May 2018 19:24:44 +0200 Subject: [PATCH 329/723] cmake: Link ext libraries after OS --- cmake/post.service.cmake | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 6c2ccd13fd..d607ebdcb8 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -205,6 +205,7 @@ if(LIBRARIES) endif() # add all extra libs +set(LIBR_CMAKE_NAMES) foreach(LIBR ${LIBRARIES}) # if relative path but not local, use includeos lib. if(NOT IS_ABSOLUTE ${LIBR} AND NOT EXISTS ${LIBR}) @@ -219,7 +220,7 @@ foreach(LIBR ${LIBRARIES}) set_target_properties(libr_${LNAME} PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(libr_${LNAME} PROPERTIES IMPORTED_LOCATION ${LIBR}) - target_link_libraries(service libr_${LNAME}) + list(APPEND LIBR_CMAKE_NAMES "libr_${LNAME}") endforeach() @@ -453,6 +454,7 @@ endif() # all the OS and C/C++ libraries + crt end target_link_libraries(service libos + ${LIBR_CMAKE_NAMES} libbotan ${OPENSSL_LIBS} libosdeps From 0bc4b6a950ec3a834a227df3c38063cad4c82be7 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 28 May 2018 19:30:34 +0200 Subject: [PATCH 330/723] cmake: Add ext libraries as whole archives --- cmake/post.service.cmake | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index d607ebdcb8..7ab0ce4d95 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -215,12 +215,8 @@ foreach(LIBR ${LIBRARIES}) set(LIBR ${OS_LIB}) endif() endif() - get_filename_component(LNAME ${LIBR} NAME_WE) - add_library(libr_${LNAME} STATIC IMPORTED) - set_target_properties(libr_${LNAME} PROPERTIES LINKER_LANGUAGE CXX) - set_target_properties(libr_${LNAME} PROPERTIES IMPORTED_LOCATION ${LIBR}) - - list(APPEND LIBR_CMAKE_NAMES "libr_${LNAME}") + # add as whole archive to allow strong symbols + list(APPEND LIBR_CMAKE_NAMES "--whole-archive ${LIBR} --no-whole-archive") endforeach() From ebaeeef47ed00a15f4e5f5eff6f3477c54e8f6d1 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Tue, 29 May 2018 09:51:44 +0200 Subject: [PATCH 331/723] Test: Changed apt to apt-get commands. --- test/net/integration/microLB/setup.sh | 2 +- test/net/integration/router/setup.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/net/integration/microLB/setup.sh b/test/net/integration/microLB/setup.sh index bb5831ad0e..6ba99baa2f 100755 --- a/test/net/integration/microLB/setup.sh +++ b/test/net/integration/microLB/setup.sh @@ -1,3 +1,3 @@ #!/bin/bash -sudo apt -qqq install -y nodejs +sudo apt-get -qqq install -y nodejs sudo -H pip -q install requests diff --git a/test/net/integration/router/setup.sh b/test/net/integration/router/setup.sh index b30af4f081..629c5d0c8c 100755 --- a/test/net/integration/router/setup.sh +++ b/test/net/integration/router/setup.sh @@ -14,7 +14,7 @@ alias server1="sudo ip netns exec $NSNAME" setup() { # TODO: it's probably not nice to install test deps here - sudo apt -qqq install -y iperf3 + sudo apt-get -qqq install -y iperf3 # Make sure the default bridge exists $INCLUDEOS_PREFIX/includeos/scripts/create_bridge.sh From a4b486454489d2b446aeb57b2072a89df5c0966f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 29 May 2018 10:13:22 +0200 Subject: [PATCH 332/723] ip6: Optimized addr using std array functionality --- api/net/ip6/addr.hpp | 47 +++++++++++++++++--------------------------- 1 file changed, 18 insertions(+), 29 deletions(-) diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index 468100e946..aeb21ec3b1 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -59,11 +59,9 @@ struct Addr { } Addr(const Addr& a) - { - for (int i = 0; i < 4; i++) { - i32[i] = a.i32[i]; - } - } + : i32{a.i32} {} + + Addr(Addr&& a) = default; // returns this IPv6 Address as a string std::string str() const { @@ -130,64 +128,55 @@ struct Addr { /** * Assignment operator */ - Addr& operator=(const Addr other) noexcept { - i32[0] = other.i32[0]; - i32[1] = other.i32[1]; - i32[2] = other.i32[2]; - i32[3] = other.i32[3]; + Addr& operator=(const Addr& other) + { + i32 = other.i32; return *this; } + Addr& operator=(Addr&& other) = default; + /** * Operator to check for equality */ - bool operator==(const Addr other) const noexcept - { return i32[0] == other.i32[0] && i32[1] == other.i32[1] && - i32[2] == other.i32[2] && i32[3] == other.i32[3]; } + bool operator==(const Addr& other) const noexcept + { return i32 == other.i32; } /** * Operator to check for inequality */ - bool operator!=(const Addr other) const noexcept + bool operator!=(const Addr& other) const noexcept { return not (*this == other); } /** * Operator to check for greater-than relationship */ - bool operator>(const Addr other) const noexcept - { - if (i32[0] > other.i32[0]) return true; - if (i32[1] > other.i32[1]) return true; - if (i32[2] > other.i32[2]) return true; - if (i32[3] > other.i32[3]) return true; - - return false; - } - + bool operator>(const Addr& other) const noexcept + { return i32 > other.i32; } /** * Operator to check for greater-than-or-equal relationship */ - bool operator>=(const Addr other) const noexcept + bool operator>=(const Addr& other) const noexcept { return (*this > other or *this == other); } /** * Operator to check for lesser-than relationship */ - bool operator<(const Addr other) const noexcept + bool operator<(const Addr& other) const noexcept { return not (*this >= other); } /** * Operator to check for lesser-than-or-equal relationship */ - bool operator<=(const Addr other) const noexcept + bool operator<=(const Addr& other) const noexcept { return (*this < other or *this == other); } /** * Operator to perform a bitwise-and operation on the given * IPv6 addresses */ - Addr operator&(const Addr other) const noexcept + Addr operator&(const Addr& other) const noexcept { return Addr{i32[0] & other.i32[0], i32[1] & other.i32[1], i32[2] & other.i32[2], @@ -221,7 +210,7 @@ struct Addr { addr[2], addr[3] }; } - Addr operator|(const Addr other) const noexcept + Addr operator|(const Addr& other) const noexcept { return Addr{i32[0] | other.i32[0], i32[1] | other.i32[1], i32[2] | other.i32[2], From fb904a35eef037e0b2fe422fc0f3f6aa39d07509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Tue, 29 May 2018 12:06:06 +0200 Subject: [PATCH 333/723] kernel: Move LiveUpdate area to high memory, unpresent identity mappings --- api/kernel/memory.hpp | 3 ++ api/kernel/os.hpp | 11 +++---- lib/LiveUpdate/CMakeLists.txt | 2 +- lib/LiveUpdate/os.cpp | 43 ++++++++++++++++++++++++++++ lib/LiveUpdate/storage.cpp | 3 +- lib/LiveUpdate/update.cpp | 11 +++++-- src/kernel/heap.cpp | 2 +- src/kernel/os.cpp | 18 ++++++------ src/platform/x86_pc/kernel_start.cpp | 4 +++ src/platform/x86_pc/softreset.cpp | 16 +++++++++-- src/plugins/system_log.cpp | 4 +-- 11 files changed, 94 insertions(+), 23 deletions(-) create mode 100644 lib/LiveUpdate/os.cpp diff --git a/api/kernel/memory.hpp b/api/kernel/memory.hpp index dc08190963..4cde19f951 100644 --- a/api/kernel/memory.hpp +++ b/api/kernel/memory.hpp @@ -310,9 +310,12 @@ namespace mem { using namespace util::bitops; const auto flags = os::mem::Access::read | os::mem::Access::write; // setup @dst as new virt area for @src + printf("mem::map\n"); os::mem::map({dst, src, flags, size}, label); // unpresent @src + printf("mem::protect\n"); os::mem::protect(src, size, os::mem::Access::none); + printf("done\n"); } }} diff --git a/api/kernel/os.hpp b/api/kernel/os.hpp index 7845708f40..cf151b2556 100644 --- a/api/kernel/os.hpp +++ b/api/kernel/os.hpp @@ -250,6 +250,12 @@ class OS { static void install_cpu_frequency(util::MHz); + /** Resume stuff from a soft reset **/ + static bool is_softreset_magic(uint32_t value); + static uintptr_t softreset_memory_end(intptr_t boot_addr); + static void resume_softreset(intptr_t boot_addr); + static void setup_liveupdate(uintptr_t phys = 0); + private: /** Process multiboot info. Called by 'start' if multibooted **/ static void multiboot(uint32_t boot_addr); @@ -259,10 +265,6 @@ class OS { /** Boot with no multiboot params */ static void legacy_boot(); - /** Resume stuff from a soft reset **/ - static bool is_softreset_magic(uint32_t value); - static void resume_softreset(intptr_t boot_addr); - static constexpr int PAGE_SHIFT = 12; static bool power_; static bool boot_sequence_passed_; @@ -274,7 +276,6 @@ class OS { static std::string version_str_; static std::string arch_str_; static uintptr_t heap_begin_; - static uintptr_t heap_end_; static uintptr_t heap_max_; static uintptr_t memory_end_; static const uintptr_t elf_binary_size_; diff --git a/lib/LiveUpdate/CMakeLists.txt b/lib/LiveUpdate/CMakeLists.txt index dbd211b027..e4b768581d 100644 --- a/lib/LiveUpdate/CMakeLists.txt +++ b/lib/LiveUpdate/CMakeLists.txt @@ -27,7 +27,7 @@ endif() # LiveUpdate static library add_library(liveupdate STATIC storage.cpp partition.cpp update.cpp resume.cpp rollback.cpp - serialize_tcp.cpp hotswap.cpp hotswap64_blob.asm + os.cpp serialize_tcp.cpp hotswap.cpp hotswap64_blob.asm ${LIU_OPENSSL_FILES} ) add_dependencies(liveupdate hotswap64) diff --git a/lib/LiveUpdate/os.cpp b/lib/LiveUpdate/os.cpp new file mode 100644 index 0000000000..c641a4e90e --- /dev/null +++ b/lib/LiveUpdate/os.cpp @@ -0,0 +1,43 @@ +#include +#include +#include + +#define HIGHMEM_LOCATION (1ull << 45) +static uintptr_t temp_phys = 0; + +//#define LIU_DEBUG 1 +#ifdef LIU_DEBUG +#define PRATTLE(fmt, ...) kprintf(fmt, ##__VA_ARGS__) +#else +#define PRATTLE(fmt, ...) /* fmt */ +#endif + +void OS::setup_liveupdate(uintptr_t phys) +{ + PRATTLE("Setting up LiveUpdate with phys at %p\n", (void*) phys); + if (phys != 0) { + PRATTLE("Deferring setup because too early\n"); + temp_phys = phys; return; + } + if (OS::liveupdate_loc_ != 0) return; + PRATTLE("New virtual move heap_max: %p\n", (void*) OS::heap_max()); + + // highmem location + OS::liveupdate_loc_ = HIGHMEM_LOCATION; + + size_t size = 0; + if (phys == 0) { + // default is 1/4 of heap from the end of memory + size = OS::heap_max() / 4; + phys = (OS::heap_max() - size) & 0xFFFFFFF0; + } + else { + size = OS::heap_max() - phys; + } + + // move location to high memory + const uintptr_t dst = (uintptr_t) OS::liveupdate_storage_area(); + PRATTLE("virtual_move %p to %p (%zu bytes)\n", + (void*) phys, (void*) dst, size); + os::mem::virtual_move(phys, size, dst, "LiveUpdate"); +} diff --git a/lib/LiveUpdate/storage.cpp b/lib/LiveUpdate/storage.cpp index cc3ece8d20..b54cf909a3 100644 --- a/lib/LiveUpdate/storage.cpp +++ b/lib/LiveUpdate/storage.cpp @@ -21,6 +21,7 @@ #include "storage.hpp" #include +#include #include #include //#define VERIFY_MEMORY @@ -111,7 +112,7 @@ void storage_header::add_end() auto& ent = create_entry(TYPE_END, 0, 0); // test against heap max - uintptr_t storage_end = (uintptr_t) ent.vla; + const auto storage_end = os::mem::virt_to_phys((uintptr_t) ent.vla); if (storage_end > OS::heap_max()) { printf("ERROR:\n" diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index c17475c95e..71dbd3713e 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -103,6 +103,7 @@ void LiveUpdate::exec(const buffer_t& blob) // the copy mechanism just copies single bytes. const char* update_area = blob.data(); char* storage_area = (char*) location; + const uintptr_t storage_area_phys = os::mem::virt_to_phys((uintptr_t) storage_area); // validate not overwriting heap, kernel area and other things if (storage_area < (char*) 0x200) { @@ -114,10 +115,12 @@ void LiveUpdate::exec(const buffer_t& blob) if (storage_area >= (char*) OS::heap_begin() && storage_area < (char*) OS::heap_end()) { throw std::runtime_error("LiveUpdate storage area is inside the heap area"); } - if (storage_area >= (char*) OS::heap_max()) { + if (storage_area_phys >= OS::heap_max()) { throw std::runtime_error("LiveUpdate storage area is outside physical memory"); } - if (storage_area >= (char*) OS::heap_max() - 0x10000) { + if (storage_area_phys >= OS::heap_max() - 0x10000) { + printf("Storage area is at %p / %p\n", + (void*) storage_area_phys, (void*) OS::heap_max()); throw std::runtime_error("LiveUpdate storage area needs at least 64kb memory"); } @@ -206,7 +209,9 @@ void LiveUpdate::exec(const buffer_t& blob) #else extern const std::pair get_rollback_location(); const auto rollback = get_rollback_location(); - void* sr_data = __os_store_soft_reset(rollback.first, rollback.second); + // we should store physical address of update location + auto rb_phys = os::mem::virt_to_phys((uintptr_t) rollback.first); + void* sr_data = __os_store_soft_reset((void*) rb_phys, rollback.second); #endif // get offsets for the new service from program header diff --git a/src/kernel/heap.cpp b/src/kernel/heap.cpp index df4924f566..99b4e53a98 100644 --- a/src/kernel/heap.cpp +++ b/src/kernel/heap.cpp @@ -74,7 +74,7 @@ void OS::init_heap(uintptr_t free_mem_begin, uintptr_t memory_end) noexcept { // NOTE: Initialize the heap before exceptions // cache-align heap, because its not aligned memory_end_ = memory_end; - heap_max_ = memory_end; + heap_max_ = memory_end-1; heap_begin_ = util::bits::roundto(free_mem_begin); auto brk_end = __init_brk(heap_begin_); __init_mmap(brk_end); diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index f06a716ba2..74508e9630 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -81,6 +81,13 @@ void* OS::liveupdate_storage_area() noexcept return (void*) OS::liveupdate_loc_; } +__attribute__((weak)) +void OS::setup_liveupdate(uintptr_t) +{ + // without LiveUpdate: storage location is at the last page? + OS::liveupdate_loc_ = OS::heap_max() & ~(uintptr_t) 0xFFF; +} + const char* OS::cmdline = nullptr; const char* OS::cmdline_args() noexcept { return cmdline; @@ -117,14 +124,9 @@ void OS::shutdown() void OS::post_start() { - // if the LiveUpdate storage area is not yet determined, - // we can assume its a fresh boot, so calculate new one based on ... - if (OS::liveupdate_loc_ == 0) - { - // default size is 1/4 of heap from the end of memory - auto size = OS::heap_max() / 4; - OS::liveupdate_loc_ = (OS::heap_max() - size) & 0xFFFFFFF0; - } + // LiveUpdate needs some initialization, although only if present + OS::setup_liveupdate(); + // Initialize the system log if plugin is present. // Dependent on the liveupdate location being set SystemLog::initialize(); diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 377b032831..870f61e9cc 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -138,6 +138,10 @@ void kernel_start(uint32_t magic, uint32_t addr) free_mem_begin = _multiboot_free_begin(addr); memory_end = _multiboot_memory_end(addr); } + else if (OS::is_softreset_magic(magic)) + { + memory_end = OS::softreset_memory_end(addr); + } PRATTLE("* Free mem begin: 0x%zx, memory end: 0x%zx \n", free_mem_begin, memory_end); diff --git a/src/platform/x86_pc/softreset.cpp b/src/platform/x86_pc/softreset.cpp index 272f9478db..573099831d 100644 --- a/src/platform/x86_pc/softreset.cpp +++ b/src/platform/x86_pc/softreset.cpp @@ -1,5 +1,6 @@ #include #include +#include #include using namespace util::literals; @@ -10,6 +11,7 @@ namespace x86 { extern uint32_t apic_timer_get_ticks() noexcept; extern void apic_timer_set_ticks(uint32_t) noexcept; } +extern char _end; struct softreset_t { @@ -30,6 +32,14 @@ bool OS::is_softreset_magic(uint32_t value) __attribute__((weak)) void softreset_service_handler(const void*, size_t) {} +uintptr_t OS::softreset_memory_end(intptr_t addr) +{ + auto* data = (softreset_t*) addr; + assert(data->high_mem > (uintptr_t) &_end); + //kprintf("Restored memory end: %p\n", data->high_mem); + return data->high_mem; +} + void OS::resume_softreset(intptr_t addr) { auto* data = (softreset_t*) addr; @@ -46,8 +56,10 @@ void OS::resume_softreset(intptr_t addr) data->checksum = csum_copy; /// restore known values - OS::liveupdate_loc_ = data->liveupdate_loc; + uintptr_t lu_phys = data->liveupdate_loc; + OS::setup_liveupdate(lu_phys); OS::memory_end_ = data->high_mem; + OS::heap_max_ = OS::memory_end_ - 1; OS::cpu_khz_ = data->cpu_freq; x86::apic_timer_set_ticks(data->apic_ticks); OS::m_is_live_updated = true; @@ -62,7 +74,7 @@ void* __os_store_soft_reset(void* extra, size_t extra_len) // store softreset data in low memory auto* data = (softreset_t*) SOFT_RESET_LOCATION; data->checksum = 0; - data->liveupdate_loc = (uintptr_t) OS::liveupdate_storage_area(); + data->liveupdate_loc = os::mem::virt_to_phys((uintptr_t) OS::liveupdate_storage_area()); data->high_mem = OS::memory_end(); data->cpu_freq = OS::cpu_freq(); data->apic_ticks = x86::apic_timer_get_ticks(); diff --git a/src/plugins/system_log.cpp b/src/plugins/system_log.cpp index 9a17d71038..6220bb7132 100644 --- a/src/plugins/system_log.cpp +++ b/src/plugins/system_log.cpp @@ -83,9 +83,9 @@ void SystemLog::initialize() using namespace util::bitops; const size_t size = MRB_LOG_SIZE - 4096; const uintptr_t syslog_area = (uintptr_t) get_ringbuffer_loc() - 4096; - const uintptr_t origin_area = (uintptr_t) OS::liveupdate_storage_area() - size; + const uintptr_t lu_phys = os::mem::virt_to_phys((uintptr_t) OS::liveupdate_storage_area()); // move systemlog to high memory and unpresent physical - os::mem::virtual_move(origin_area, size, syslog_area, "SystemLog"); + os::mem::virtual_move(lu_phys - size, size, syslog_area, "SystemLog"); #endif auto& buffer = get_log_buffer(); From 9b5d871de2ff88c2d698b7fdfb99df119e587f97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Tue, 29 May 2018 12:07:08 +0200 Subject: [PATCH 334/723] test: Add setup.sh for GRUB test --- test/kernel/integration/grub/setup.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 test/kernel/integration/grub/setup.sh diff --git a/test/kernel/integration/grub/setup.sh b/test/kernel/integration/grub/setup.sh new file mode 100755 index 0000000000..3a0f4c1c51 --- /dev/null +++ b/test/kernel/integration/grub/setup.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sudo apt install -y grub-pc xorriso From 274b052153caa24e270a41d84c906c0f151f2fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Tue, 29 May 2018 12:07:08 +0200 Subject: [PATCH 335/723] test: Add setup.sh for GRUB test --- test/kernel/integration/grub/setup.sh | 2 ++ 1 file changed, 2 insertions(+) create mode 100755 test/kernel/integration/grub/setup.sh diff --git a/test/kernel/integration/grub/setup.sh b/test/kernel/integration/grub/setup.sh new file mode 100755 index 0000000000..3a0f4c1c51 --- /dev/null +++ b/test/kernel/integration/grub/setup.sh @@ -0,0 +1,2 @@ +#!/bin/bash +sudo apt install -y grub-pc xorriso From bf3b25d55b65425436ca3a296bf02c57fb2ef286 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Thu, 17 May 2018 14:50:49 +0530 Subject: [PATCH 336/723] ip6: ndp re-factoring. Insert ndp into icmp6 --- api/net/ip6/addr.hpp | 11 ++ api/net/ip6/icmp6.hpp | 8 +- api/net/ip6/ndp.hpp | 216 +++++++++++++++++++++++++++++++++++ api/net/ip6/packet_icmp6.hpp | 5 + src/CMakeLists.txt | 2 +- src/net/ip6/icmp6.cpp | 91 +-------------- src/net/ip6/ndp.cpp | 114 +++++++++++++++++- 7 files changed, 355 insertions(+), 92 deletions(-) create mode 100644 api/net/ip6/ndp.hpp diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index 468100e946..d60bee7c4d 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -239,4 +239,15 @@ struct Addr { } __attribute__((packed)); //< struct Addr } //< namespace ip6 } //< namespace net + +// Allow an IPv6 address to be used as key in e.g. std::unordered_map +namespace std { + template<> + struct hash { + size_t operator()(const net::ip6::Addr& addr) const { + // This is temporary. Use a proper hash + return std::hash{}(addr.i64[0] + addr.i64[1]); + } + }; +} //< namespace std #endif diff --git a/api/net/ip6/icmp6.hpp b/api/net/ip6/icmp6.hpp index c779d33d06..945764964b 100644 --- a/api/net/ip6/icmp6.hpp +++ b/api/net/ip6/icmp6.hpp @@ -20,6 +20,7 @@ #ifndef NET_IP6_ICMPv6_HPP #define NET_IP6_ICMPv6_HPP +#include "ndp.hpp" #include "packet_icmp6.hpp" #include #include @@ -109,7 +110,10 @@ namespace net // Delegate output to network layer inline void set_network_out(downstream s) - { network_layer_out_ = s; }; + { + network_layer_out_ = s; + ndp_.set_network_out(s); + }; /** * Destination Unreachable sent from host because of port (UDP) or protocol (IP6) unreachable @@ -145,10 +149,12 @@ namespace net void ping(const std::string& hostname); void ping(const std::string& hostname, icmp_func callback, int sec_wait = SEC_WAIT_FOR_REPLY); + Ndp& ndp() { return ndp_; } private: static int request_id_; // message identifier for messages originating from IncludeOS Stack& inet_; + Ndp ndp_; downstream network_layer_out_ = nullptr; uint8_t includeos_payload_[48] = {'I','N','C','L','U','D', 'E','O','S','1','2','3','4','5', diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp new file mode 100644 index 0000000000..936d7dbb03 --- /dev/null +++ b/api/net/ip6/ndp.hpp @@ -0,0 +1,216 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#ifndef NET_IP6_NDP_HPP +#define NET_IP6_NDP_HPP + +#include +#include +#include +#include "ip6.hpp" +#include "packet_icmp6.hpp" + +using namespace std::chrono_literals; +namespace net { + + class ICMPv6; + class PacketNdp; + + /** NDP manager, including an NDP-Cache. */ + class Ndp { + + public: + using Stack = IP6::Stack; + using Route_checker = delegate; + using Ndp_resolver = delegate; + using ICMP_type = ICMP6_error::ICMP_type; + + enum Opcode { H_request = 0x100, H_reply = 0x200 }; + + /** Ndp opcodes (Big-endian) */ + static constexpr uint16_t H_htype_eth {0x0100}; + static constexpr uint16_t H_ptype_ip6 {0x0008}; + static constexpr uint16_t H_hlen_plen {0x0406}; + + /** Number of resolution retries **/ + static constexpr int ndp_retries = 3; + + /** Constructor */ + explicit Ndp(Stack&) noexcept; + + struct __attribute__((packed)) header { + uint16_t htype; // Hardware type + uint16_t ptype; // Protocol type + uint16_t hlen_plen; // Protocol address length + uint16_t opcode; // Opcode + MAC::Addr shwaddr; // Source mac + IP6::addr sipaddr; // Source ip + MAC::Addr dhwaddr; // Target mac + IP6::addr dipaddr; // Target ip + }; + + /** Handle incoming NDP packet. */ + void receive(icmp6::Packet& pckt); + int receive_neighbor_solicitation(icmp6::Packet& pckt); + void send_router_solicitation(); + + /** Roll your own ndp-resolution system. */ + void set_resolver(Ndp_resolver ar) + { ndp_resolver_ = ar; } + + enum Resolver_name { DEFAULT, HH_MAP }; + + /** + * Set NDP proxy policy. + * No route checker (default) implies NDP proxy functionality is disabled. + * + * @param delg : delegate to determine if we should reply to a given IP + */ + void set_proxy_policy(Route_checker delg) + { proxy_ = delg; } + + /** Delegate link-layer output. */ + void set_linklayer_out(downstream_link link) + { linklayer_out_ = link; } + + /** Downstream transmission. */ + void transmit(Packet_ptr, IP6::addr next_hop); + + /** Cache IP resolution. */ + void cache(IP6::addr, MAC::Addr); + + /** Flush the NDP cache. RFC-2.3.2.1 */ + void flush_cache() + { cache_.clear(); }; + + /** Flush expired cache entries. RFC-2.3.2.1 */ + void flush_expired (); + + void set_cache_flush_interval(std::chrono::minutes m) { + flush_interval_ = m; + } + + // Delegate output to network layer + inline void set_network_out(downstream s) + { network_layer_out_ = s; }; + + private: + + /** NDP cache expires after cache_exp_sec_ seconds */ + static constexpr uint16_t cache_exp_sec_ {60 * 5}; + + /** Cache entries are just MAC's and timestamps */ + struct Cache_entry { + /** Map needs empty constructor (we have no emplace yet) */ + Cache_entry() noexcept = default; + + Cache_entry(MAC::Addr mac) noexcept + : mac_(mac), timestamp_(RTC::time_since_boot()) {} + + Cache_entry(const Cache_entry& cpy) noexcept + : mac_(cpy.mac_), timestamp_(cpy.timestamp_) {} + + void update() noexcept { timestamp_ = RTC::time_since_boot(); } + + bool expired() const noexcept + { return RTC::time_since_boot() > timestamp_ + cache_exp_sec_; } + + MAC::Addr mac() const noexcept + { return mac_; } + + RTC::timestamp_t timestamp() const noexcept + { return timestamp_; } + + RTC::timestamp_t expires() const noexcept + { return timestamp_ + cache_exp_sec_; } + + private: + MAC::Addr mac_; + RTC::timestamp_t timestamp_; + }; //< struct Cache_entry + + struct Queue_entry { + Packet_ptr pckt; + int tries_remaining = ndp_retries; + + Queue_entry(Packet_ptr p) + : pckt{std::move(p)} + {} + }; + + using Cache = std::unordered_map; + using PacketQueue = std::unordered_map; + + + /** Stats */ + uint32_t& requests_rx_; + uint32_t& requests_tx_; + uint32_t& replies_rx_; + uint32_t& replies_tx_; + + std::chrono::minutes flush_interval_ = 5min; + + Timer resolve_timer_ {{ *this, &Ndp::resolve_waiting }}; + Timer flush_timer_ {{ *this, &Ndp::flush_expired }}; + + Stack& inet_; + Route_checker proxy_ = nullptr; + downstream network_layer_out_ = nullptr; + + // Needs to know which mac address to put in header->swhaddr + MAC::Addr mac_; + + // Outbound data goes through here */ + downstream_link linklayer_out_ = nullptr; + + // The NDP cache + Cache cache_ {}; + + // RFC-1122 2.3.2.2 Packet queue + PacketQueue waiting_packets_; + + // Settable resolver - defualts to ndp_resolve + Ndp_resolver ndp_resolver_ = {this, &Ndp::ndp_resolve}; + + /** Respond to ndp request */ + void ndp_respond(header* hdr_in, IP6::addr ack_ip); + + /** Send an ndp resolution request */ + void ndp_resolve(IP6::addr next_hop); + + /** + * Add a packet to waiting queue, to be sent when IP is resolved. + * + * Implements RFC1122 + * 2.3.2.1 : Prevent NDP flooding + * 2.3.2.2 : Packets SHOULD be queued. + */ + void await_resolution(Packet_ptr, IP6::addr); + + /** Create a default initialized NDP-packet */ + Packet_ptr create_packet(); + + /** Retry ndp-resolution for packets still waiting */ + void resolve_waiting(); + + + }; //< class Ndp + +} //< namespace net + +#endif //< NET_NDP_HPP diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index 22387e131b..b30e5d56f2 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -16,6 +16,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +#pragma once +#ifndef PACKET_ICMP6_HPP +#define PACKET_ICMP6_HPP + #include #include #include @@ -291,3 +295,4 @@ namespace icmp6 { }; } } +#endif //< PACKET_ICMP6_HPP diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0d73bb64b3..b6bad93cea 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,7 +48,7 @@ set(OS_OBJECTS net/tcp/write_queue.cpp net/tcp/rttm.cpp net/tcp/listener.cpp net/tcp/read_buffer.cpp net/tcp/read_request.cpp net/tcp/stream.cpp net/ip4/icmp4.cpp net/ip4/udp.cpp net/ip4/udp_socket.cpp - net/ip6/ip6.cpp net/ip6/icmp6.cpp + net/ip6/ip6.cpp net/ip6/icmp6.cpp net/ip6/ndp.cpp net/dns/dns.cpp net/dns/client.cpp net/dhcp/dh4client.cpp net/dhcp/dhcpd.cpp net/buffer_store.cpp net/inet.cpp net/super_stack.cpp net/configure.cpp net/conntrack.cpp net/vlan_manager.cpp diff --git a/src/net/ip6/icmp6.cpp b/src/net/ip6/icmp6.cpp index 5d9218bea3..aaa1368cc6 100644 --- a/src/net/ip6/icmp6.cpp +++ b/src/net/ip6/icmp6.cpp @@ -52,87 +52,9 @@ namespace net int ICMPv6::request_id_ = 0; ICMPv6::ICMPv6(Stack& inet) : - inet_{inet} + inet_{inet}, ndp_(inet) {} - void ICMPv6::send_router_solicitation() - { - icmp6::Packet req(inet_.ip6_packet_factory()); - req.ip().set_ip_src(inet_.ip6_addr()); - req.ip().set_ip_dst(ip6::Addr::node_all_nodes); - req.ip().set_ip_hop_limit(255); - req.set_type(ICMP_type::ND_ROUTER_SOLICATION); - req.set_code(0); - req.set_reserved(0); - // Set multicast addreqs - // IPv6mcast_02: 33:33:00:00:00:02 - - // Add checksum - req.set_checksum(); - - PRINT(" Router solicit size: %i payload size: %i, checksum: 0x%x\n", - req.ip().size(), req.payload().size(), req.compute_checksum()); - - network_layer_out_(req.release()); - } - - int ICMPv6::receive_neighbor_solicitation(icmp6::Packet& req) - { - bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; - IP6::addr target = req.neighbor_sol().get_target(); - - PRINT("ICMPv6 NDP Neighbor solicitation request\n"); - PRINT(">> target: %s\n", target.str().c_str()); - - if (target.is_multicast()) { - PRINT("ND: neighbor solictation target address is multicast\n"); - return -1; - } - - icmp6::Packet res(inet_.ip6_packet_factory()); - - // drop if the packet is too small - if (res.ip().capacity() < IP6_HEADER_LEN - + (int) res.header_size() + req.payload().size()) - { - PRINT("WARNING: Network MTU too small for ICMP response, dropping\n"); - return -1; - } - - // Populate response IP header - res.ip().set_ip_src(inet_.ip6_addr()); - if (any_src) { - res.ip().set_ip_dst(ip6::Addr::node_all_nodes); - } else { - res.ip().set_ip_dst(req.ip().ip_src()); - } - res.ip().set_ip_hop_limit(255); - - // Populate response ICMP header - res.set_type(ICMP_type::ND_NEIGHBOR_ADV); - res.set_code(0); - res.set_neighbor_adv_flag(NEIGH_ADV_SOL | NEIGH_ADV_OVERRIDE); - PRINT(" Transmitting Neighbor adv to %s\n", - res.ip().ip_dst().str().c_str()); - - // Insert target link address, ICMP6 option header and our mac address - // TODO: This is hacky. Fix it - MAC::Addr dest_mac("c0:01:0a:00:00:2a"); - // Target link address - res.set_payload({req.payload().data(), 16 }); - res.set_ndp_options_header(0x02, 0x01); - res.set_payload({reinterpret_cast (&dest_mac), 6}); - - // Add checksum - res.set_checksum(); - - PRINT(" Neighbor Adv Response size: %i payload size: %i, checksum: 0x%x\n", - res.ip().size(), res.payload().size(), res.compute_checksum()); - - network_layer_out_(res.release()); - return 0; - } - void ICMPv6::receive(Packet_ptr pckt) { if (not is_full_header((size_t) pckt->size())) // Drop if not a full header return; @@ -168,20 +90,11 @@ namespace net PRINT(" ICMP multicast message from %s\n", req.ip().ip_src().str().c_str()); break; case (ICMP_type::ND_ROUTER_SOLICATION): - PRINT(" ICMP Router solictation message from %s\n", req.ip().ip_src().str().c_str()); - break; case (ICMP_type::ND_ROUTER_ADV): - PRINT(" ICMP Router advertisement message from %s\n", req.ip().ip_src().str().c_str()); - break; case (ICMP_type::ND_NEIGHBOR_SOL): - PRINT(" ICMP Neigbor solictation message from %s\n", req.ip().ip_src().str().c_str()); - receive_neighbor_solicitation(req); - break; case (ICMP_type::ND_NEIGHBOR_ADV): - PRINT(" ICMP Neigbor advertisement message from %s\n", req.ip().ip_src().str().c_str()); - break; case (ICMP_type::ND_REDIRECT): - PRINT(" ICMP Neigbor redirect message from %s\n", req.ip().ip_src().str().c_str()); + ndp().receive(req); break; case (ICMP_type::ROUTER_RENUMBERING): PRINT(" ICMP Router re-numbering message from %s\n", req.ip().ip_src().str().c_str()); diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index ae4928fd7d..d3d9e2be1d 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -15,4 +15,116 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include +//#define NDP_DEBUG 1 +#ifdef NDP_DEBUG +#define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define PRINT(fmt, ...) /* fmt */ +#endif +#include +#include + +namespace net +{ + int Ndp::receive_neighbor_solicitation(icmp6::Packet& req) + { + bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; + IP6::addr target = req.neighbor_sol().get_target(); + + PRINT("ICMPv6 NDP Neighbor solicitation request\n"); + PRINT(">> target: %s\n", target.str().c_str()); + + if (target.is_multicast()) { + PRINT("ND: neighbor solictation target address is multicast\n"); + return -1; + } + + icmp6::Packet res(inet_.ip6_packet_factory()); + + // drop if the packet is too small + if (res.ip().capacity() < IP6_HEADER_LEN + + (int) res.header_size() + req.payload().size()) + { + PRINT("WARNING: Network MTU too small for ICMP response, dropping\n"); + return -1; + } + + // Populate response IP header + res.ip().set_ip_src(inet_.ip6_addr()); + if (any_src) { + res.ip().set_ip_dst(ip6::Addr::node_all_nodes); + } else { + res.ip().set_ip_dst(req.ip().ip_src()); + } + res.ip().set_ip_hop_limit(255); + + // Populate response ICMP header + res.set_type(ICMP_type::ND_NEIGHBOR_ADV); + res.set_code(0); + res.set_neighbor_adv_flag(NEIGH_ADV_SOL | NEIGH_ADV_OVERRIDE); + PRINT(" Transmitting Neighbor adv to %s\n", + res.ip().ip_dst().str().c_str()); + + // Insert target link address, ICMP6 option header and our mac address + // TODO: This is hacky. Fix it + MAC::Addr dest_mac("c0:01:0a:00:00:2a"); + // Target link address + res.set_payload({req.payload().data(), 16 }); + res.set_ndp_options_header(0x02, 0x01); + res.set_payload({reinterpret_cast (&dest_mac), 6}); + + // Add checksum + res.set_checksum(); + + PRINT(" Neighbor Adv Response size: %i payload size: %i, checksum: 0x%x\n", + res.ip().size(), res.payload().size(), res.compute_checksum()); + + network_layer_out_(res.release()); + return 0; + } + + void Ndp::send_router_solicitation() + { + icmp6::Packet req(inet_.ip6_packet_factory()); + req.ip().set_ip_src(inet_.ip6_addr()); + req.ip().set_ip_dst(ip6::Addr::node_all_nodes); + req.ip().set_ip_hop_limit(255); + req.set_type(ICMP_type::ND_ROUTER_SOLICATION); + req.set_code(0); + req.set_reserved(0); + // Set multicast addreqs + // IPv6mcast_02: 33:33:00:00:00:02 + + // Add checksum + req.set_checksum(); + + PRINT(" Router solicit size: %i payload size: %i, checksum: 0x%x\n", + req.ip().size(), req.payload().size(), req.compute_checksum()); + + network_layer_out_(req.release()); + } + + + void Ndp::receive(icmp6::Packet& pckt) { + switch(pckt.type()) { + case (ICMP_type::ND_ROUTER_SOLICATION): + PRINT(" ICMP Router solictation message from %s\n", pckt.ip().ip_src().str().c_str()); + break; + case (ICMP_type::ND_ROUTER_ADV): + PRINT(" ICMP Router advertisement message from %s\n", pckt.ip().ip_src().str().c_str()); + break; + case (ICMP_type::ND_NEIGHBOR_SOL): + PRINT(" ICMP Neigbor solictation message from %s\n", pckt.ip().ip_src().str().c_str()); + receive_neighbor_solicitation(pckt); + break; + case (ICMP_type::ND_NEIGHBOR_ADV): + PRINT(" ICMP Neigbor advertisement message from %s\n", pckt.ip().ip_src().str().c_str()); + break; + case (ICMP_type::ND_REDIRECT): + PRINT(" ICMP Neigbor redirect message from %s\n", pckt.ip().ip_src().str().c_str()); + break; + default: + return; + } + } +} From 30ef0ca47ee5ca3e72223d2ebdef4b02a7131499 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Thu, 17 May 2018 16:36:37 +0530 Subject: [PATCH 337/723] ndp: packet parsing boilerplate API --- api/net/ip6/addr.hpp | 8 +++++++ api/net/ip6/ndp.hpp | 10 ++++++++- src/net/ip6/ndp.cpp | 50 ++++++++++++++++++++++++++++++-------------- 3 files changed, 51 insertions(+), 17 deletions(-) diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index d60bee7c4d..cb0c5a524b 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -120,6 +120,14 @@ struct Addr { return ((ntohs(i16[0]) & 0xFF00) == 0xFF00); } + bool is_solicit_multicast() const + { + return ((ntohs(i32[0]) ^ (0xff020000)) | + ntohs(i32[1]) | + (ntohs(i32[2]) ^ (0x00000001)) | + (ntohs(i32[3]) ^ 0xff)) == 0; + } + /** * **/ diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp index 936d7dbb03..ce80e40ec9 100644 --- a/api/net/ip6/ndp.hpp +++ b/api/net/ip6/ndp.hpp @@ -66,8 +66,16 @@ namespace net { /** Handle incoming NDP packet. */ void receive(icmp6::Packet& pckt); - int receive_neighbor_solicitation(icmp6::Packet& pckt); + void receive_neighbor_solicitation(icmp6::Packet& pckt); + void receive_neighbor_advertisement(icmp6::Packet& pckt); + void receive_router_solicitation(icmp6::Packet& pckt); + void receive_router_advertisement(icmp6::Packet& pckt); + + /** Send out NDP packet */ + void send_neighbor_solicitation(); + void send_neighbor_advertisement(icmp6::Packet& req); void send_router_solicitation(); + void send_router_advertisement(); /** Roll your own ndp-resolution system. */ void set_resolver(Ndp_resolver ar) diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index d3d9e2be1d..51dd170bfe 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -21,32 +21,22 @@ #else #define PRINT(fmt, ...) /* fmt */ #endif -#include +#include #include namespace net { - int Ndp::receive_neighbor_solicitation(icmp6::Packet& req) + void Ndp::send_neighbor_advertisement(icmp6::Packet& req) { - bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; - IP6::addr target = req.neighbor_sol().get_target(); - - PRINT("ICMPv6 NDP Neighbor solicitation request\n"); - PRINT(">> target: %s\n", target.str().c_str()); - - if (target.is_multicast()) { - PRINT("ND: neighbor solictation target address is multicast\n"); - return -1; - } - icmp6::Packet res(inet_.ip6_packet_factory()); + bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; // drop if the packet is too small if (res.ip().capacity() < IP6_HEADER_LEN + (int) res.header_size() + req.payload().size()) { PRINT("WARNING: Network MTU too small for ICMP response, dropping\n"); - return -1; + return; } // Populate response IP header @@ -80,7 +70,36 @@ namespace net res.ip().size(), res.payload().size(), res.compute_checksum()); network_layer_out_(res.release()); - return 0; + } + + void Ndp::receive_neighbor_advertisement(icmp6::Packet& req) + { + } + + void Ndp::send_neighbor_solicitation() + { + } + + void Ndp::receive_neighbor_solicitation(icmp6::Packet& req) + { + bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; + IP6::addr target = req.neighbor_sol().get_target(); + + PRINT("ICMPv6 NDP Neighbor solicitation request\n"); + PRINT(">> target: %s\n", target.str().c_str()); + + if (target.is_multicast()) { + PRINT("ND: neighbor solictation target address is multicast\n"); + return; + } + + if (any_src && !req.ip().ip_dst().is_solicit_multicast()) { + PRINT("ND: neighbor solictation address is any source " + "but not solicit destination\n"); + return; + } + + return send_neighbor_advertisement(req); } void Ndp::send_router_solicitation() @@ -104,7 +123,6 @@ namespace net network_layer_out_(req.release()); } - void Ndp::receive(icmp6::Packet& pckt) { switch(pckt.type()) { case (ICMP_type::ND_ROUTER_SOLICATION): From 24fe711241e0bb4cb730eb9fe4abeff9e23645bd Mon Sep 17 00:00:00 2001 From: niks3089 Date: Thu, 17 May 2018 23:52:20 +0530 Subject: [PATCH 338/723] ndp: Add ndp options --- api/net/ip6/ndp.hpp | 1 - api/net/ip6/packet_icmp6.hpp | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp index ce80e40ec9..4f32ebf21a 100644 --- a/api/net/ip6/ndp.hpp +++ b/api/net/ip6/ndp.hpp @@ -29,7 +29,6 @@ using namespace std::chrono_literals; namespace net { class ICMPv6; - class PacketNdp; /** NDP manager, including an NDP-Cache. */ class Ndp { diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index b30e5d56f2..ca910362b7 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -41,6 +41,22 @@ namespace icmp6 { uint16_t sequence; }; + enum class NdpOpt : uint8_t { + ND_OPT_PREFIX_INFO_END = 0, + ND_OPT_SOURCE_LL_ADDR = 1, /* RFC2461 */ + ND_OPT_TARGET_LL_ADDR = 2, /* RFC2461 */ + ND_OPT_PREFIX_INFO = 3, /* RFC2461 */ + ND_OPT_REDIRECT_HDR = 4, /* RFC2461 */ + ND_OPT_MTU = 5, /* RFC2461 */ + ND_OPT_NONCE = 14, /* RFC7527 */ + ND_OPT_ARRAY_MAX, + ND_OPT_ROUTE_INFO = 24, /* RFC4191 */ + ND_OPT_RDNSS = 25, /* RFC5006 */ + ND_OPT_DNSSL = 31, /* RFC6106 */ + ND_OPT_6CO = 34, /* RFC6775 */ + ND_OPT_MAX + }; + struct Header { Type type; uint8_t code; From ac062534b28b7f3fcd68414f8a37c84a728f3540 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Fri, 18 May 2018 15:12:48 +0530 Subject: [PATCH 339/723] ndp: Have a separate class for ndp packets and let it reside inside icmp6 --- api/net/ip6/packet_icmp6.hpp | 214 +++++++++++++++++++++-------------- src/net/ip6/icmp6.cpp | 1 + src/net/ip6/ndp.cpp | 20 +++- 3 files changed, 146 insertions(+), 89 deletions(-) diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index ca910362b7..4034d32611 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -36,27 +36,133 @@ namespace icmp6 { class Packet { + class NdpPacket { + + private: + Packet& icmp6_; + + enum class NdpOpt : uint8_t { + ND_OPT_SOURCE_LL_ADDR = 1, /* RFC2461 */ + ND_OPT_TARGET_LL_ADDR = 2, /* RFC2461 */ + ND_OPT_PREFIX_INFO = 3, /* RFC2461 */ + ND_OPT_REDIRECT_HDR = 4, /* RFC2461 */ + ND_OPT_MTU = 5, /* RFC2461 */ + ND_OPT_NONCE = 14, /* RFC7527 */ + ND_OPT_ARRAY_MAX, + ND_OPT_ROUTE_INFO = 24, /* RFC4191 */ + ND_OPT_RDNSS = 25, /* RFC5006 */ + ND_OPT_DNSSL = 31, /* RFC6106 */ + ND_OPT_6CO = 34, /* RFC6775 */ + ND_OPT_MAX + }; + + struct nd_options_header { + uint8_t type; + uint8_t len; + } __attribute__((packed)); + + class NdpOptions { + + private: + struct nd_options_header *header_; +#if 0 + std::array (NdpOpt::ND_OPT_ARRAY_MAX)> opt_array; + struct nd_options_header &user_opts; + struct nd_options_header &user_opts_end; +#endif + + public: + NdpOptions(uint8_t &opt) { + header_ = reinterpret_cast(opt); + } + + void parse() + { + } + }; + + struct RouterSol + { + uint8_t options[0]; + } __attribute__((packed)); + + struct RouterAdv + { + uint32_t reachable_time; + uint32_t retrans_timer; + } __attribute__((packed)); + + struct RouterRedirect + { + IP6::addr target; + IP6::addr dest; + uint8_t options[0]; + } __attribute__((packed)); + + struct NeighborSol + { + IP6::addr target; + uint8_t options[0]; + + IP6::addr get_target() + { return target; } + + bool parse_options(uint8_t options) + { + struct NdpOptions opt(options); + opt.parse(); + } + + } __attribute__((packed)); + + struct NeighborAdv + { + IP6::addr target; + uint8_t options[0]; + + void set_target(IP6::addr tar) + { target = tar; } + } __attribute__((packed)); + + public: + + NdpPacket(Packet& icmp6) : icmp6_(icmp6) {} + + RouterSol& router_sol() + { return *reinterpret_cast(&(icmp6_.header().payload[0])); } + + RouterAdv& router_adv() + { return *reinterpret_cast(&(icmp6_.header().payload[0])); } + + RouterRedirect& router_redirect() + { return *reinterpret_cast(&(icmp6_.header().payload[0])); } + + NeighborSol& neighbor_sol() + { return *reinterpret_cast(&(icmp6_.header().payload[0])); } + + NeighborAdv& neighbor_adv() + { return *reinterpret_cast(&(icmp6_.header().payload[0])); } + + void set_neighbor_adv_flag(uint32_t flag) + { icmp6_.header().rso_flags = htonl(flag << 28); } + + void set_ndp_options_header(uint8_t type, uint8_t len) + { + struct nd_options_header header; + header.type = type; + header.len = len; + + icmp6_.set_payload({reinterpret_cast(&header), + sizeof header}); + } + }; + struct IdSe { uint16_t identifier; uint16_t sequence; }; - enum class NdpOpt : uint8_t { - ND_OPT_PREFIX_INFO_END = 0, - ND_OPT_SOURCE_LL_ADDR = 1, /* RFC2461 */ - ND_OPT_TARGET_LL_ADDR = 2, /* RFC2461 */ - ND_OPT_PREFIX_INFO = 3, /* RFC2461 */ - ND_OPT_REDIRECT_HDR = 4, /* RFC2461 */ - ND_OPT_MTU = 5, /* RFC2461 */ - ND_OPT_NONCE = 14, /* RFC7527 */ - ND_OPT_ARRAY_MAX, - ND_OPT_ROUTE_INFO = 24, /* RFC4191 */ - ND_OPT_RDNSS = 25, /* RFC5006 */ - ND_OPT_DNSSL = 31, /* RFC6106 */ - ND_OPT_6CO = 34, /* RFC6775 */ - ND_OPT_MAX - }; - struct Header { Type type; uint8_t code; @@ -84,48 +190,6 @@ namespace icmp6 { uint8_t next; } __attribute__((packed)); - struct nd_options_header { - uint8_t type; - uint8_t len; - } __attribute__((packed)); - - struct RouterSol - { - uint8_t options[0]; - } __attribute__((packed)); - - struct RouterAdv - { - uint32_t reachable_time; - uint32_t retrans_timer; - } __attribute__((packed)); - - struct RouterRedirect - { - IP6::addr target; - IP6::addr dest; - uint8_t options[0]; - } __attribute__((packed)); - - struct NeighborSol - { - IP6::addr target; - uint8_t options[0]; - - IP6::addr get_target() - { return target; } - - } __attribute__((packed)); - - struct NeighborAdv - { - IP6::addr target; - uint8_t options[0]; - - void set_target(IP6::addr tar) - { target = tar; } - } __attribute__((packed)); - public: using Span = gsl::span; @@ -165,21 +229,6 @@ namespace icmp6 { Span header_and_data() { return {pckt_->layer_begin(), pckt_->ip_header_len() + 8}; } - RouterSol& router_sol() - { return *reinterpret_cast(&(header().payload[0])); } - - RouterAdv& router_adv() - { return *reinterpret_cast(&(header().payload[0])); } - - RouterRedirect& router_redirect() - { return *reinterpret_cast(&(header().payload[0])); } - - NeighborSol& neighbor_sol() - { return *reinterpret_cast(&(header().payload[0])); } - - NeighborAdv& neighbor_adv() - { return *reinterpret_cast(&(header().payload[0])); } - void set_type(Type t) noexcept { header().type = t; } @@ -267,17 +316,6 @@ namespace icmp6 { header().checksum = compute_checksum();; } - void set_neighbor_adv_flag(uint32_t flag) - { header().rso_flags = htonl(flag << 28); } - - void set_ndp_options_header(uint8_t type, uint8_t len) - { - struct nd_options_header ndo; - ndo.type = type; - ndo.len = len; - set_payload({reinterpret_cast (&ndo), sizeof ndo}); - } - void set_payload(Span new_load) { pckt_->set_data_end(pckt_->ip_header_len() + header_size() + payload_offset_ + new_load.size()); @@ -291,12 +329,12 @@ namespace icmp6 { /** Construct from existing packet **/ Packet(IP6::IP_packet_ptr pckt) - : pckt_{ std::move(pckt) }, payload_offset_{0} + : pckt_{ std::move(pckt) }, ndp_(*this), payload_offset_{0} { } /** Provision fresh packet from factory **/ Packet(IP6::IP_packet_factory create) - : pckt_{create(Protocol::ICMPv6)}, payload_offset_{0} + : pckt_{create(Protocol::ICMPv6)}, ndp_(*this), payload_offset_{0} { pckt_->increment_data_end(sizeof(Header)); } @@ -305,8 +343,12 @@ namespace icmp6 { IP6::IP_packet_ptr release() { return std::move(pckt_); } + NdpPacket ndp() + { return ndp_; } + private: IP6::IP_packet_ptr pckt_; + NdpPacket ndp_; uint16_t payload_offset_; }; } diff --git a/src/net/ip6/icmp6.cpp b/src/net/ip6/icmp6.cpp index aaa1368cc6..f529aefa14 100644 --- a/src/net/ip6/icmp6.cpp +++ b/src/net/ip6/icmp6.cpp @@ -22,6 +22,7 @@ #define PRINT(fmt, ...) /* fmt */ #endif #include +#include #include #include diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index 51dd170bfe..f4f666f66f 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -23,9 +23,19 @@ #endif #include #include +#include namespace net { + Ndp::Ndp(Stack& inet) noexcept: + requests_rx_ {Statman::get().create(Stat::UINT32, inet.ifname() + ".ndp.requests_rx").get_uint32()}, + requests_tx_ {Statman::get().create(Stat::UINT32, inet.ifname() + ".ndp.requests_tx").get_uint32()}, + replies_rx_ {Statman::get().create(Stat::UINT32, inet.ifname() + ".ndp.replies_rx").get_uint32()}, + replies_tx_ {Statman::get().create(Stat::UINT32, inet.ifname() + ".ndp.replies_tx").get_uint32()}, + inet_ {inet}, + mac_ (inet.link_addr()) + {} + void Ndp::send_neighbor_advertisement(icmp6::Packet& req) { icmp6::Packet res(inet_.ip6_packet_factory()); @@ -51,7 +61,7 @@ namespace net // Populate response ICMP header res.set_type(ICMP_type::ND_NEIGHBOR_ADV); res.set_code(0); - res.set_neighbor_adv_flag(NEIGH_ADV_SOL | NEIGH_ADV_OVERRIDE); + res.ndp().set_neighbor_adv_flag(NEIGH_ADV_SOL | NEIGH_ADV_OVERRIDE); PRINT(" Transmitting Neighbor adv to %s\n", res.ip().ip_dst().str().c_str()); @@ -60,7 +70,7 @@ namespace net MAC::Addr dest_mac("c0:01:0a:00:00:2a"); // Target link address res.set_payload({req.payload().data(), 16 }); - res.set_ndp_options_header(0x02, 0x01); + res.ndp().set_ndp_options_header(0x02, 0x01); res.set_payload({reinterpret_cast (&dest_mac), 6}); // Add checksum @@ -83,7 +93,7 @@ namespace net void Ndp::receive_neighbor_solicitation(icmp6::Packet& req) { bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; - IP6::addr target = req.neighbor_sol().get_target(); + IP6::addr target = req.ndp().neighbor_sol().get_target(); PRINT("ICMPv6 NDP Neighbor solicitation request\n"); PRINT(">> target: %s\n", target.str().c_str()); @@ -145,4 +155,8 @@ namespace net return; } } + + void Ndp::resolve_waiting() {} + void Ndp::flush_expired() {} + void Ndp::ndp_resolve(IP6::addr next_hop) {} } From 0369003338c00ecf0f6b641d71acbd4092b23ce0 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Fri, 18 May 2018 21:52:24 +0530 Subject: [PATCH 340/723] ndp: Parsing apis for ndp options --- api/net/ip6/packet_icmp6.hpp | 50 ++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index 4034d32611..64a78e685a 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -36,11 +36,10 @@ namespace icmp6 { class Packet { + using ICMP_type = ICMP6_error::ICMP_type; class NdpPacket { private: - Packet& icmp6_; - enum class NdpOpt : uint8_t { ND_OPT_SOURCE_LL_ADDR = 1, /* RFC2461 */ ND_OPT_TARGET_LL_ADDR = 2, /* RFC2461 */ @@ -59,6 +58,7 @@ namespace icmp6 { struct nd_options_header { uint8_t type; uint8_t len; + uint8_t payload[0]; } __attribute__((packed)); class NdpOptions { @@ -73,12 +73,20 @@ namespace icmp6 { #endif public: - NdpOptions(uint8_t &opt) { + NdpOptions() : header_{NULL} {} + + void parse() + { + } + + bool setup_options(uint8_t *opt) + { header_ = reinterpret_cast(opt); } - void parse() + struct nd_options_header *get_header(uint8_t &opt) { + return reinterpret_cast(opt); } }; @@ -98,6 +106,7 @@ namespace icmp6 { IP6::addr target; IP6::addr dest; uint8_t options[0]; + } __attribute__((packed)); struct NeighborSol @@ -110,8 +119,6 @@ namespace icmp6 { bool parse_options(uint8_t options) { - struct NdpOptions opt(options); - opt.parse(); } } __attribute__((packed)); @@ -123,11 +130,38 @@ namespace icmp6 { void set_target(IP6::addr tar) { target = tar; } + } __attribute__((packed)); + Packet& icmp6_; + NdpOptions ndp_opt_; + public: - NdpPacket(Packet& icmp6) : icmp6_(icmp6) {} + NdpPacket(Packet& icmp6) : icmp6_(icmp6) { + } + + bool setup(icmp6::Type type) { + switch(type) { + case (ICMP_type::ND_ROUTER_SOLICATION): + ndp_opt_.setup_options(router_sol().options); + break; + case (ICMP_type::ND_ROUTER_ADV): + break; + case (ICMP_type::ND_NEIGHBOR_SOL): + ndp_opt_.setup_options(neighbor_sol().options); + break; + case (ICMP_type::ND_NEIGHBOR_ADV): + ndp_opt_.setup_options(neighbor_adv().options); + break; + case (ICMP_type::ND_REDIRECT): + ndp_opt_.setup_options(router_redirect().options); + break; + default: + return false; + } + return true; + } RouterSol& router_sol() { return *reinterpret_cast(&(icmp6_.header().payload[0])); } @@ -343,7 +377,7 @@ namespace icmp6 { IP6::IP_packet_ptr release() { return std::move(pckt_); } - NdpPacket ndp() + NdpPacket& ndp() { return ndp_; } private: From 71bb38ea1bc600280932bd2d79c0c3754f8467e7 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Mon, 21 May 2018 15:50:29 +0530 Subject: [PATCH 341/723] ndp: Support for parsing ndp options. --- api/net/ip6/packet_icmp6.hpp | 122 +++++++++++++++++++---------------- src/net/ip6/ndp.cpp | 121 +++++++++++++++++++++++++++++++++- 2 files changed, 186 insertions(+), 57 deletions(-) diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index 64a78e685a..84aebf7c2b 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -34,27 +34,28 @@ namespace net { namespace icmp6 { + enum { + ND_OPT_PREFIX_INFO_END = 0, + ND_OPT_SOURCE_LL_ADDR = 1, /* RFC2461 */ + ND_OPT_TARGET_LL_ADDR = 2, /* RFC2461 */ + ND_OPT_PREFIX_INFO = 3, /* RFC2461 */ + ND_OPT_REDIRECT_HDR = 4, /* RFC2461 */ + ND_OPT_MTU = 5, /* RFC2461 */ + ND_OPT_NONCE = 14, /* RFC7527 */ + ND_OPT_ARRAY_MAX, + ND_OPT_ROUTE_INFO = 24, /* RFC4191 */ + ND_OPT_RDNSS = 25, /* RFC5006 */ + ND_OPT_DNSSL = 31, /* RFC6106 */ + ND_OPT_6CO = 34, /* RFC6775 */ + ND_OPT_MAX + }; + class Packet { using ICMP_type = ICMP6_error::ICMP_type; class NdpPacket { private: - enum class NdpOpt : uint8_t { - ND_OPT_SOURCE_LL_ADDR = 1, /* RFC2461 */ - ND_OPT_TARGET_LL_ADDR = 2, /* RFC2461 */ - ND_OPT_PREFIX_INFO = 3, /* RFC2461 */ - ND_OPT_REDIRECT_HDR = 4, /* RFC2461 */ - ND_OPT_MTU = 5, /* RFC2461 */ - ND_OPT_NONCE = 14, /* RFC7527 */ - ND_OPT_ARRAY_MAX, - ND_OPT_ROUTE_INFO = 24, /* RFC4191 */ - ND_OPT_RDNSS = 25, /* RFC5006 */ - ND_OPT_DNSSL = 31, /* RFC6106 */ - ND_OPT_6CO = 34, /* RFC6775 */ - ND_OPT_MAX - }; - struct nd_options_header { uint8_t type; uint8_t len; @@ -65,40 +66,58 @@ namespace icmp6 { private: struct nd_options_header *header_; -#if 0 - std::array (NdpOpt::ND_OPT_ARRAY_MAX)> opt_array; - struct nd_options_header &user_opts; - struct nd_options_header &user_opts_end; -#endif - - public: - NdpOptions() : header_{NULL} {} + std::array opt_array; + struct nd_options_header *nd_opts_ri; + struct nd_options_header *nd_opts_ri_end; + struct nd_options_header *user_opts; + struct nd_options_header *user_opts_end; - void parse() + bool is_useropt(struct nd_options_header *opt) { + if (opt->type == ND_OPT_RDNSS || + opt->type == ND_OPT_DNSSL) { + return true; + } + return false; } - bool setup_options(uint8_t *opt) - { - header_ = reinterpret_cast(opt); - } + public: + NdpOptions() : header_{NULL} {} + void parse(uint8_t *opt, uint16_t opts_len); struct nd_options_header *get_header(uint8_t &opt) { return reinterpret_cast(opt); } + + uint8_t *get_option_data(uint8_t option) + { + if (option < ND_OPT_ARRAY_MAX) { + if (opt_array[option]) { + return static_cast (opt_array[option]->payload); + } + } + return NULL; + } }; struct RouterSol { uint8_t options[0]; + + uint16_t option_offset() + { return 0; } + } __attribute__((packed)); struct RouterAdv { - uint32_t reachable_time; - uint32_t retrans_timer; + uint32_t reachable_time; + uint32_t retrans_timer; + + uint16_t option_offset() + { return 0; } + } __attribute__((packed)); struct RouterRedirect @@ -107,6 +126,9 @@ namespace icmp6 { IP6::addr dest; uint8_t options[0]; + uint16_t option_offset() + { return 16 * 2; } + } __attribute__((packed)); struct NeighborSol @@ -117,9 +139,8 @@ namespace icmp6 { IP6::addr get_target() { return target; } - bool parse_options(uint8_t options) - { - } + uint16_t option_offset() + { return 16; } } __attribute__((packed)); @@ -131,6 +152,9 @@ namespace icmp6 { void set_target(IP6::addr tar) { target = tar; } + uint16_t option_offset() + { return 16; } + } __attribute__((packed)); Packet& icmp6_; @@ -141,27 +165,7 @@ namespace icmp6 { NdpPacket(Packet& icmp6) : icmp6_(icmp6) { } - bool setup(icmp6::Type type) { - switch(type) { - case (ICMP_type::ND_ROUTER_SOLICATION): - ndp_opt_.setup_options(router_sol().options); - break; - case (ICMP_type::ND_ROUTER_ADV): - break; - case (ICMP_type::ND_NEIGHBOR_SOL): - ndp_opt_.setup_options(neighbor_sol().options); - break; - case (ICMP_type::ND_NEIGHBOR_ADV): - ndp_opt_.setup_options(neighbor_adv().options); - break; - case (ICMP_type::ND_REDIRECT): - ndp_opt_.setup_options(router_redirect().options); - break; - default: - return false; - } - return true; - } + void parse(icmp6::Type type); RouterSol& router_sol() { return *reinterpret_cast(&(icmp6_.header().payload[0])); } @@ -190,6 +194,11 @@ namespace icmp6 { icmp6_.set_payload({reinterpret_cast(&header), sizeof header}); } + + uint8_t* get_option_data(int opt) + { + return ndp_opt_.get_option_data(opt); + } }; struct IdSe { @@ -246,6 +255,9 @@ namespace icmp6 { uint16_t sequence() const noexcept { return header().idse.sequence; } + uint16_t payload_len() const noexcept + { return pckt_->data_end() - (pckt_->layer_begin() + pckt_->ip_header_len() + header_size()); } + /** * Where the payload of an ICMP packet starts, calculated from the start of the IP header * The payload of an ICMP error message packet contains the original packet sent that caused an diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index f4f666f66f..a80c70a4d5 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -94,6 +94,8 @@ namespace net { bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; IP6::addr target = req.ndp().neighbor_sol().get_target(); + uint8_t *lladdr, *nonce_opt; + uint64_t nonce = 0; PRINT("ICMPv6 NDP Neighbor solicitation request\n"); PRINT(">> target: %s\n", target.str().c_str()); @@ -108,8 +110,36 @@ namespace net "but not solicit destination\n"); return; } + req.ndp().parse(ICMP_type::ND_NEIGHBOR_SOL); + lladdr = req.ndp().get_option_data(icmp6::ND_OPT_SOURCE_LL_ADDR); + + if (lladdr) { + if (any_src) { + PRINT("ND: bad any source packet with link layer option\n"); + return; + } + } + + nonce_opt = req.ndp().get_option_data(icmp6::ND_OPT_NONCE); + if (nonce_opt) { + //memcpy(&nonce, nonce_opt, 6); + } + + bool is_dest_multicast = req.ip().ip_dst().is_multicast(); + + if (target != inet_.ip6_addr()) { + /* Not for us. Should we forward? */ + return; + } + + if (any_src) { + send_neighbor_advertisement(req); + return; + } + + /* Update/Create cache entry for the source address */ + send_neighbor_advertisement(req); - return send_neighbor_advertisement(req); } void Ndp::send_router_solicitation() @@ -159,4 +189,91 @@ namespace net void Ndp::resolve_waiting() {} void Ndp::flush_expired() {} void Ndp::ndp_resolve(IP6::addr next_hop) {} -} + + // NDP packet function definitions + namespace icmp6 { + void Packet::NdpPacket::parse(icmp6::Type type) + { + switch(type) { + case (ICMP_type::ND_ROUTER_SOLICATION): + ndp_opt_.parse(router_sol().options, + (icmp6_.payload_len() - router_sol().option_offset())); + break; + case (ICMP_type::ND_ROUTER_ADV): + break; + case (ICMP_type::ND_NEIGHBOR_SOL): + ndp_opt_.parse(neighbor_sol().options, + (icmp6_.payload_len() - neighbor_sol().option_offset())); + break; + case (ICMP_type::ND_NEIGHBOR_ADV): + ndp_opt_.parse(neighbor_adv().options, + (icmp6_.payload_len() - neighbor_adv().option_offset())); + break; + case (ICMP_type::ND_REDIRECT): + ndp_opt_.parse(router_redirect().options, + (icmp6_.payload_len() - router_redirect().option_offset())); + break; + default: + break; + } + } + + void Packet::NdpPacket::NdpOptions::parse(uint8_t *opt, uint16_t opts_len) + { + uint16_t opt_len; + header_ = reinterpret_cast(opt); + struct nd_options_header *option_hdr = header_; + + if (option_hdr == NULL) { + return; + } + while(opts_len) { + if (opts_len < sizeof (struct nd_options_header)) { + return; + } + opt_len = option_hdr->len << 3; + + if (opts_len < opt_len || opt_len == 0) { + return; + } + switch (option_hdr->type) { + case ND_OPT_SOURCE_LL_ADDR: + case ND_OPT_TARGET_LL_ADDR: + case ND_OPT_MTU: + case ND_OPT_NONCE: + case ND_OPT_REDIRECT_HDR: + if (opt_array[option_hdr->type]) { + } else { + opt_array[option_hdr->type] = option_hdr; + } + break; + case ND_OPT_PREFIX_INFO: + opt_array[ND_OPT_PREFIX_INFO_END] = option_hdr; + if (!opt_array[ND_OPT_PREFIX_INFO]) { + opt_array[ND_OPT_PREFIX_INFO] = option_hdr; + } + break; + case ND_OPT_ROUTE_INFO: + nd_opts_ri_end = option_hdr; + if (!nd_opts_ri) { + nd_opts_ri = option_hdr; + } + break; + default: + if (is_useropt(option_hdr)) { + user_opts_end = option_hdr; + if (!user_opts) { + user_opts = option_hdr; + } + } else { + PRINT("%s: Unsupported option: type=%d, len=%d\n", + __FUNCTION__, option_hdr->type, option_hdr->len); + } + } + opts_len -= opt_len; + option_hdr = (option_hdr + opt_len); + } + } + + } // icmp6 +} // net From 79ea943f6fddfed9cad01aa85a85345d767edfd2 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Mon, 21 May 2018 22:05:18 +0530 Subject: [PATCH 342/723] ndp: initial caching code & ndp wiring --- api/hw/mac_addr.hpp | 5 ++ api/net/ip6/icmp6.hpp | 12 ++-- api/net/ip6/ndp.hpp | 27 ++------ src/net/inet.cpp | 2 + src/net/ip6/ndp.cpp | 152 ++++++++++++++++++++++++++++++++++++++---- 5 files changed, 159 insertions(+), 39 deletions(-) diff --git a/api/hw/mac_addr.hpp b/api/hw/mac_addr.hpp index a50a98bb39..17a6c94893 100644 --- a/api/hw/mac_addr.hpp +++ b/api/hw/mac_addr.hpp @@ -87,6 +87,11 @@ union Addr { memcpy(part, macaddr, PARTS_LEN); } + Addr(uint8_t *addr) noexcept + { + memcpy(part, addr, PARTS_LEN); + } + /** * Assignment operator * diff --git a/api/net/ip6/icmp6.hpp b/api/net/ip6/icmp6.hpp index 945764964b..b2da5add7d 100644 --- a/api/net/ip6/icmp6.hpp +++ b/api/net/ip6/icmp6.hpp @@ -110,10 +110,14 @@ namespace net // Delegate output to network layer inline void set_network_out(downstream s) - { - network_layer_out_ = s; - ndp_.set_network_out(s); - }; + { + network_layer_out_ = s; + } + + void ndp_transmit(Packet_ptr ptr, IP6::addr next_hop) + { + //ndp_.transmit(ptr, next_hop); + } /** * Destination Unreachable sent from host because of port (UDP) or protocol (IP6) unreachable diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp index 4f32ebf21a..cf384c94af 100644 --- a/api/net/ip6/ndp.hpp +++ b/api/net/ip6/ndp.hpp @@ -39,30 +39,12 @@ namespace net { using Ndp_resolver = delegate; using ICMP_type = ICMP6_error::ICMP_type; - enum Opcode { H_request = 0x100, H_reply = 0x200 }; - - /** Ndp opcodes (Big-endian) */ - static constexpr uint16_t H_htype_eth {0x0100}; - static constexpr uint16_t H_ptype_ip6 {0x0008}; - static constexpr uint16_t H_hlen_plen {0x0406}; - /** Number of resolution retries **/ static constexpr int ndp_retries = 3; /** Constructor */ explicit Ndp(Stack&) noexcept; - struct __attribute__((packed)) header { - uint16_t htype; // Hardware type - uint16_t ptype; // Protocol type - uint16_t hlen_plen; // Protocol address length - uint16_t opcode; // Opcode - MAC::Addr shwaddr; // Source mac - IP6::addr sipaddr; // Source ip - MAC::Addr dhwaddr; // Target mac - IP6::addr dipaddr; // Target ip - }; - /** Handle incoming NDP packet. */ void receive(icmp6::Packet& pckt); void receive_neighbor_solicitation(icmp6::Packet& pckt); @@ -99,7 +81,10 @@ namespace net { void transmit(Packet_ptr, IP6::addr next_hop); /** Cache IP resolution. */ - void cache(IP6::addr, MAC::Addr); + bool cache(IP6::addr ip, MAC::Addr mac, uint8_t flags); + + /** Lookup for cache entry */ + bool lookup(bool create, IP6::addr ip, uint8_t *ll_addr, uint8_t flags); /** Flush the NDP cache. RFC-2.3.2.1 */ void flush_cache() @@ -179,7 +164,6 @@ namespace net { Route_checker proxy_ = nullptr; downstream network_layer_out_ = nullptr; - // Needs to know which mac address to put in header->swhaddr MAC::Addr mac_; // Outbound data goes through here */ @@ -194,9 +178,6 @@ namespace net { // Settable resolver - defualts to ndp_resolve Ndp_resolver ndp_resolver_ = {this, &Ndp::ndp_resolve}; - /** Respond to ndp request */ - void ndp_respond(header* hdr_in, IP6::addr ack_ip); - /** Send an ndp resolution request */ void ndp_resolve(IP6::addr next_hop); diff --git a/src/net/inet.cpp b/src/net/inet.cpp index 05d7aeab3c..6ed4923dd9 100644 --- a/src/net/inet.cpp +++ b/src/net/inet.cpp @@ -76,6 +76,7 @@ Inet::Inet(hw::Nic& nic) /** Downstream delegates */ auto link_top(nic_.create_link_downstream()); auto arp_top(IP4::downstream_arp{arp_, &Arp::transmit}); + auto ndp_top(IP6::downstream_ndp{icmp6_, &ICMPv6::ndp_transmit}); auto ip4_top(downstream{ip4_, &IP4::transmit}); auto ip6_top(downstream{ip6_, &IP6::transmit}); @@ -98,6 +99,7 @@ Inet::Inet(hw::Nic& nic) // IP6 -> Link ip6_.set_linklayer_out(link_top); + //ip6_.set_linklayer_out(ndp_top); // UDP6 -> IP6 // udp6->set_network_out(ip6_top); diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index a80c70a4d5..9945da4bee 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -95,6 +95,7 @@ namespace net bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; IP6::addr target = req.ndp().neighbor_sol().get_target(); uint8_t *lladdr, *nonce_opt; + bool found; uint64_t nonce = 0; PRINT("ICMPv6 NDP Neighbor solicitation request\n"); @@ -122,10 +123,10 @@ namespace net nonce_opt = req.ndp().get_option_data(icmp6::ND_OPT_NONCE); if (nonce_opt) { - //memcpy(&nonce, nonce_opt, 6); + //memcpy(&nonce, nonce_opt, 6); } - bool is_dest_multicast = req.ip().ip_dst().is_multicast(); + bool is_dest_multicast = req.ip().ip_dst().is_multicast(); if (target != inet_.ip6_addr()) { /* Not for us. Should we forward? */ @@ -138,7 +139,12 @@ namespace net } /* Update/Create cache entry for the source address */ - send_neighbor_advertisement(req); + found = lookup(!is_dest_multicast || lladdr, + req.ip().ip_src(), lladdr, 0); + + if (found) { + send_neighbor_advertisement(req); + } } @@ -186,23 +192,145 @@ namespace net } } - void Ndp::resolve_waiting() {} - void Ndp::flush_expired() {} - void Ndp::ndp_resolve(IP6::addr next_hop) {} + bool Ndp::lookup(bool create, IP6::addr ip, uint8_t *ll_addr, uint8_t flags) + { + auto entry = cache_.find(ip); + if (entry != cache_.end()) { + return true; + } else if (create == true && ll_addr) { + MAC::Addr mac(ll_addr); + cache(ip, mac, flags); + return true; + } + return false; + } + + bool Ndp::cache(IP6::addr ip, MAC::Addr mac, uint8_t flags) + { + auto entry = cache_.find(ip); + if (entry != cache_.end()) { + PRINT("Cached entry found: %s recorded @ %zu. Updating timestamp\n", + entry->second.mac().str().c_str(), entry->second.timestamp()); + if (entry->second.mac() != mac) { + cache_.erase(entry); + cache_[ip] = mac; + } else { + entry->second.update(); + } + } else { + cache_[ip] = mac; // Insert + if (UNLIKELY(not flush_timer_.is_running())) { + flush_timer_.start(flush_interval_); + } + } + } + + void Ndp::resolve_waiting() + { + PRINT(" resolve timer doing sweep\n"); + + for (auto it =waiting_packets_.begin(); it != waiting_packets_.end();){ + if (it->second.tries_remaining--) { + ndp_resolver_(it->first); + it++; + } else { + it = waiting_packets_.erase(it); + } + } + + if (not waiting_packets_.empty()) + resolve_timer_.start(1s); + + } + + void Ndp::await_resolution(Packet_ptr pckt, IP6::addr next_hop) { + auto queue = waiting_packets_.find(next_hop); + PRINT(" Waiting for resolution of %s\n", next_hop.str().c_str()); + if (queue != waiting_packets_.end()) { + PRINT("\t * Packets already queueing for this IP\n"); + queue->second.pckt->chain(std::move(pckt)); + } else { + PRINT("\t *This is the first packet going to that IP\n"); + waiting_packets_.emplace(std::make_pair(next_hop, Queue_entry{std::move(pckt)})); + + // Try resolution immediately + ndp_resolver_(next_hop); + + // Retry later + resolve_timer_.start(1s); + } + } + + void Ndp::flush_expired() + { + PRINT(" Flushing expired entries\n"); + std::vector expired; + for (auto ent : cache_) { + if (ent.second.expired()) { + expired.push_back(ent.first); + } + } + + for (auto ip : expired) { + cache_.erase(ip); + } + + if (not cache_.empty()) { + flush_timer_.start(flush_interval_); + } + } + + void Ndp::ndp_resolve(IP6::addr next_hop) + { + PRINT(" %s\n", next_hop.str().c_str()); + + // Stat increment requests sent + requests_tx_++; + + // Send ndp solicit + } + + void Ndp::transmit(Packet_ptr pckt, IP6::addr next_hop) + { + + Expects(pckt->size()); + + PRINT(" physical> Transmitting %u bytes to %s\n", + (uint32_t) pckt->size(), next_hop.str().c_str()); + + MAC::Addr dest_mac; + + // If we don't have a cached IP, perform address resolution + auto cache_entry = cache_.find(next_hop); + if (UNLIKELY(cache_entry == cache_.end())) { + PRINT(" No cache entry for IP %s. Resolving. \n", next_hop.to_string().c_str()); + await_resolution(std::move(pckt), next_hop); + return; + } + + // Get MAC from cache + dest_mac = cache_[next_hop].mac(); + + PRINT(" Found cache entry for IP %s -> %s \n", + next_hop.to_string().c_str(), dest_mac.to_string().c_str()); + + // Move chain to linklayer + linklayer_out_(std::move(pckt), dest_mac, Ethertype::IP4); + } // NDP packet function definitions namespace icmp6 { - void Packet::NdpPacket::parse(icmp6::Type type) + void Packet::NdpPacket::parse(icmp6::Type type) { switch(type) { case (ICMP_type::ND_ROUTER_SOLICATION): - ndp_opt_.parse(router_sol().options, + ndp_opt_.parse(router_sol().options, (icmp6_.payload_len() - router_sol().option_offset())); break; case (ICMP_type::ND_ROUTER_ADV): break; case (ICMP_type::ND_NEIGHBOR_SOL): - ndp_opt_.parse(neighbor_sol().options, + ndp_opt_.parse(neighbor_sol().options, (icmp6_.payload_len() - neighbor_sol().option_offset())); break; case (ICMP_type::ND_NEIGHBOR_ADV): @@ -218,10 +346,10 @@ namespace net } } - void Packet::NdpPacket::NdpOptions::parse(uint8_t *opt, uint16_t opts_len) + void Packet::NdpPacket::NdpOptions::parse(uint8_t *opt, uint16_t opts_len) { uint16_t opt_len; - header_ = reinterpret_cast(opt); + header_ = reinterpret_cast(opt); struct nd_options_header *option_hdr = header_; if (option_hdr == NULL) { @@ -267,7 +395,7 @@ namespace net } } else { PRINT("%s: Unsupported option: type=%d, len=%d\n", - __FUNCTION__, option_hdr->type, option_hdr->len); + __FUNCTION__, option_hdr->type, option_hdr->len); } } opts_len -= opt_len; From bc617aed4734114cfee3e0d60dbea1ebee3c5a55 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Tue, 22 May 2018 17:10:15 +0530 Subject: [PATCH 343/723] ndp: Fix the ndp cache crash. Adding more debug to find out why the de-ref on ndp options are failing --- api/net/ip6/icmp6.hpp | 2 ++ api/net/ip6/ndp.hpp | 2 +- api/net/ip6/packet_icmp6.hpp | 13 ++++++++--- src/net/ip6/ndp.cpp | 42 +++++++++++++++++++++++++----------- 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/api/net/ip6/icmp6.hpp b/api/net/ip6/icmp6.hpp index b2da5add7d..b2c38b6c3a 100644 --- a/api/net/ip6/icmp6.hpp +++ b/api/net/ip6/icmp6.hpp @@ -112,6 +112,8 @@ namespace net inline void set_network_out(downstream s) { network_layer_out_ = s; + // TODO: Remove this. NDP packets should be sent directly on link + ndp_.set_network_out(s); } void ndp_transmit(Packet_ptr ptr, IP6::addr next_hop) diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp index cf384c94af..75e639df77 100644 --- a/api/net/ip6/ndp.hpp +++ b/api/net/ip6/ndp.hpp @@ -81,7 +81,7 @@ namespace net { void transmit(Packet_ptr, IP6::addr next_hop); /** Cache IP resolution. */ - bool cache(IP6::addr ip, MAC::Addr mac, uint8_t flags); + void cache(IP6::addr ip, MAC::Addr mac, uint8_t flags); /** Lookup for cache entry */ bool lookup(bool create, IP6::addr ip, uint8_t *ll_addr, uint8_t flags); diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index 84aebf7c2b..4423d369da 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -66,7 +66,8 @@ namespace icmp6 { private: struct nd_options_header *header_; - std::array opt_array; + //std::array opt_array; + struct nd_options_header* opt_array[ND_OPT_ARRAY_MAX]; struct nd_options_header *nd_opts_ri; struct nd_options_header *nd_opts_ri_end; struct nd_options_header *user_opts; @@ -82,7 +83,8 @@ namespace icmp6 { } public: - NdpOptions() : header_{NULL} {} + NdpOptions() : header_{NULL}, nd_opts_ri{NULL}, + nd_opts_ri_end{NULL}, user_opts{NULL}, user_opts_end{NULL} {} void parse(uint8_t *opt, uint16_t opts_len); struct nd_options_header *get_header(uint8_t &opt) @@ -94,6 +96,11 @@ namespace icmp6 { { if (option < ND_OPT_ARRAY_MAX) { if (opt_array[option]) { + printf("Returning options, %d, %d\n", + opt_array[option]->type, opt_array[option]->len); + char mac[6]; + memcpy(mac, opt_array[option]->payload, 6); + printf("lladdres is api is %s\n", mac); return static_cast (opt_array[option]->payload); } } @@ -162,7 +169,7 @@ namespace icmp6 { public: - NdpPacket(Packet& icmp6) : icmp6_(icmp6) { + NdpPacket(Packet& icmp6) : icmp6_(icmp6), ndp_opt_() { } void parse(icmp6::Type type); diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index 9945da4bee..c068e3a782 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -15,14 +15,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -//#define NDP_DEBUG 1 +#define NDP_DEBUG 1 #ifdef NDP_DEBUG #define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else #define PRINT(fmt, ...) /* fmt */ #endif -#include +#include #include +#include +#include #include namespace net @@ -62,7 +64,7 @@ namespace net res.set_type(ICMP_type::ND_NEIGHBOR_ADV); res.set_code(0); res.ndp().set_neighbor_adv_flag(NEIGH_ADV_SOL | NEIGH_ADV_OVERRIDE); - PRINT(" Transmitting Neighbor adv to %s\n", + PRINT(" Transmitting Neighbor adv to %s\n", res.ip().ip_dst().str().c_str()); // Insert target link address, ICMP6 option header and our mac address @@ -76,7 +78,7 @@ namespace net // Add checksum res.set_checksum(); - PRINT(" Neighbor Adv Response size: %i payload size: %i, checksum: 0x%x\n", + PRINT(" Neighbor Adv Response size: %i payload size: %i, checksum: 0x%x\n", res.ip().size(), res.payload().size(), res.compute_checksum()); network_layer_out_(res.release()); @@ -119,6 +121,10 @@ namespace net PRINT("ND: bad any source packet with link layer option\n"); return; } + char mac[6]; + memcpy(mac, lladdr, 6); + PRINT("printing lladdr\n"); + PRINT("lladdres is %s\n", mac); } nonce_opt = req.ndp().get_option_data(icmp6::ND_OPT_NONCE); @@ -163,7 +169,7 @@ namespace net // Add checksum req.set_checksum(); - PRINT(" Router solicit size: %i payload size: %i, checksum: 0x%x\n", + PRINT(" Router solicit size: %i payload size: %i, checksum: 0x%x\n", req.ip().size(), req.payload().size(), req.compute_checksum()); network_layer_out_(req.release()); @@ -172,20 +178,20 @@ namespace net void Ndp::receive(icmp6::Packet& pckt) { switch(pckt.type()) { case (ICMP_type::ND_ROUTER_SOLICATION): - PRINT(" ICMP Router solictation message from %s\n", pckt.ip().ip_src().str().c_str()); + PRINT(" ICMP Router solictation message from %s\n", pckt.ip().ip_src().str().c_str()); break; case (ICMP_type::ND_ROUTER_ADV): - PRINT(" ICMP Router advertisement message from %s\n", pckt.ip().ip_src().str().c_str()); + PRINT(" ICMP Router advertisement message from %s\n", pckt.ip().ip_src().str().c_str()); break; case (ICMP_type::ND_NEIGHBOR_SOL): - PRINT(" ICMP Neigbor solictation message from %s\n", pckt.ip().ip_src().str().c_str()); + PRINT(" ICMP Neigbor solictation message from %s\n", pckt.ip().ip_src().str().c_str()); receive_neighbor_solicitation(pckt); break; case (ICMP_type::ND_NEIGHBOR_ADV): - PRINT(" ICMP Neigbor advertisement message from %s\n", pckt.ip().ip_src().str().c_str()); + PRINT(" ICMP Neigbor advertisement message from %s\n", pckt.ip().ip_src().str().c_str()); break; case (ICMP_type::ND_REDIRECT): - PRINT(" ICMP Neigbor redirect message from %s\n", pckt.ip().ip_src().str().c_str()); + PRINT(" ICMP Neigbor redirect message from %s\n", pckt.ip().ip_src().str().c_str()); break; default: return; @@ -205,24 +211,29 @@ namespace net return false; } - bool Ndp::cache(IP6::addr ip, MAC::Addr mac, uint8_t flags) + void Ndp::cache(IP6::addr ip, MAC::Addr mac, uint8_t flags) { + PRINT("Ndp Caching IP %s for %s\n", ip.str().c_str(), mac.str().c_str()); auto entry = cache_.find(ip); + PRINT("Finding cache\n"); if (entry != cache_.end()) { PRINT("Cached entry found: %s recorded @ %zu. Updating timestamp\n", entry->second.mac().str().c_str(), entry->second.timestamp()); if (entry->second.mac() != mac) { cache_.erase(entry); - cache_[ip] = mac; + cache_.emplace(ip, mac); } else { entry->second.update(); } } else { - cache_[ip] = mac; // Insert + PRINT("Trying to add cache\n"); + cache_.emplace(ip, mac); // Insert + PRINT("Done adding cache\n"); if (UNLIKELY(not flush_timer_.is_running())) { flush_timer_.start(flush_interval_); } } + PRINT("Done with cache\n"); } void Ndp::resolve_waiting() @@ -370,10 +381,15 @@ namespace net case ND_OPT_MTU: case ND_OPT_NONCE: case ND_OPT_REDIRECT_HDR: + printf("Setting option header: %d\n", option_hdr->type); if (opt_array[option_hdr->type]) { } else { opt_array[option_hdr->type] = option_hdr; } + printf("De-referencing option headerd\n"); + option_hdr = opt_array[option_hdr->type]; + printf("De-referencing option headerd\n"); + printf("De-referencing option header: %d\n", option_hdr->type); break; case ND_OPT_PREFIX_INFO: opt_array[ND_OPT_PREFIX_INFO_END] = option_hdr; From 973354b7d7ec64c445a07fcbcf05f627feab4417 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Tue, 22 May 2018 19:22:47 +0530 Subject: [PATCH 344/723] ndp: Fix ndp option de-ref crash --- api/net/ip6/packet_icmp6.hpp | 22 ++++++++-------------- src/net/ip6/ndp.cpp | 18 +++--------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index 4423d369da..d32f16d22c 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -66,14 +66,13 @@ namespace icmp6 { private: struct nd_options_header *header_; - //std::array opt_array; - struct nd_options_header* opt_array[ND_OPT_ARRAY_MAX]; + std::array opt_array; struct nd_options_header *nd_opts_ri; struct nd_options_header *nd_opts_ri_end; struct nd_options_header *user_opts; struct nd_options_header *user_opts_end; - bool is_useropt(struct nd_options_header *opt) + bool is_useropt(struct nd_options_header *opt) { if (opt->type == ND_OPT_RDNSS || opt->type == ND_OPT_DNSSL) { @@ -83,11 +82,11 @@ namespace icmp6 { } public: - NdpOptions() : header_{NULL}, nd_opts_ri{NULL}, - nd_opts_ri_end{NULL}, user_opts{NULL}, user_opts_end{NULL} {} + NdpOptions() : header_{nullptr}, opt_array{}, nd_opts_ri{nullptr}, + nd_opts_ri_end{nullptr}, user_opts{nullptr}, user_opts_end{nullptr} {} void parse(uint8_t *opt, uint16_t opts_len); - struct nd_options_header *get_header(uint8_t &opt) + struct nd_options_header *get_header(uint8_t &opt) { return reinterpret_cast(opt); } @@ -96,11 +95,6 @@ namespace icmp6 { { if (option < ND_OPT_ARRAY_MAX) { if (opt_array[option]) { - printf("Returning options, %d, %d\n", - opt_array[option]->type, opt_array[option]->len); - char mac[6]; - memcpy(mac, opt_array[option]->payload, 6); - printf("lladdres is api is %s\n", mac); return static_cast (opt_array[option]->payload); } } @@ -172,7 +166,7 @@ namespace icmp6 { NdpPacket(Packet& icmp6) : icmp6_(icmp6), ndp_opt_() { } - void parse(icmp6::Type type); + void parse(icmp6::Type type); RouterSol& router_sol() { return *reinterpret_cast(&(icmp6_.header().payload[0])); } @@ -198,13 +192,13 @@ namespace icmp6 { header.type = type; header.len = len; - icmp6_.set_payload({reinterpret_cast(&header), + icmp6_.set_payload({reinterpret_cast(&header), sizeof header}); } uint8_t* get_option_data(int opt) { - return ndp_opt_.get_option_data(opt); + return ndp_opt_.get_option_data(opt); } }; diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index c068e3a782..b622bb4868 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#define NDP_DEBUG 1 +//#define NDP_DEBUG 1 #ifdef NDP_DEBUG #define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) #else @@ -101,7 +101,7 @@ namespace net uint64_t nonce = 0; PRINT("ICMPv6 NDP Neighbor solicitation request\n"); - PRINT(">> target: %s\n", target.str().c_str()); + PRINT("target: %s\n", target.str().c_str()); if (target.is_multicast()) { PRINT("ND: neighbor solictation target address is multicast\n"); @@ -121,10 +121,6 @@ namespace net PRINT("ND: bad any source packet with link layer option\n"); return; } - char mac[6]; - memcpy(mac, lladdr, 6); - PRINT("printing lladdr\n"); - PRINT("lladdres is %s\n", mac); } nonce_opt = req.ndp().get_option_data(icmp6::ND_OPT_NONCE); @@ -215,7 +211,6 @@ namespace net { PRINT("Ndp Caching IP %s for %s\n", ip.str().c_str(), mac.str().c_str()); auto entry = cache_.find(ip); - PRINT("Finding cache\n"); if (entry != cache_.end()) { PRINT("Cached entry found: %s recorded @ %zu. Updating timestamp\n", entry->second.mac().str().c_str(), entry->second.timestamp()); @@ -226,14 +221,11 @@ namespace net entry->second.update(); } } else { - PRINT("Trying to add cache\n"); cache_.emplace(ip, mac); // Insert - PRINT("Done adding cache\n"); if (UNLIKELY(not flush_timer_.is_running())) { flush_timer_.start(flush_interval_); } } - PRINT("Done with cache\n"); } void Ndp::resolve_waiting() @@ -256,7 +248,7 @@ namespace net void Ndp::await_resolution(Packet_ptr pckt, IP6::addr next_hop) { auto queue = waiting_packets_.find(next_hop); - PRINT(" Waiting for resolution of %s\n", next_hop.str().c_str()); + PRINT(" Waiting for resolution of %s\n", next_hop.str().c_str()); if (queue != waiting_packets_.end()) { PRINT("\t * Packets already queueing for this IP\n"); queue->second.pckt->chain(std::move(pckt)); @@ -381,15 +373,11 @@ namespace net case ND_OPT_MTU: case ND_OPT_NONCE: case ND_OPT_REDIRECT_HDR: - printf("Setting option header: %d\n", option_hdr->type); if (opt_array[option_hdr->type]) { } else { opt_array[option_hdr->type] = option_hdr; } - printf("De-referencing option headerd\n"); option_hdr = opt_array[option_hdr->type]; - printf("De-referencing option headerd\n"); - printf("De-referencing option header: %d\n", option_hdr->type); break; case ND_OPT_PREFIX_INFO: opt_array[ND_OPT_PREFIX_INFO_END] = option_hdr; From 9ae9d4f978dda08edd32d57e05cbf9ef9d76d4f5 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Wed, 23 May 2018 10:28:13 +0530 Subject: [PATCH 345/723] arp: Replace cache insertion using [] with emplace --- src/net/ip4/arp.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net/ip4/arp.cpp b/src/net/ip4/arp.cpp index 148e344267..b6f3d24475 100644 --- a/src/net/ip4/arp.cpp +++ b/src/net/ip4/arp.cpp @@ -111,13 +111,13 @@ namespace net { if (entry->second.mac() != mac) { cache_.erase(entry); - cache_[ip] = mac; + cache_.emplace(ip, mac); } else { entry->second.update(); } } else { - cache_[ip] = mac; // Insert + cache_.emplace(ip, mac); // Insert if (UNLIKELY(not flush_timer_.is_running())) { flush_timer_.start(flush_interval_); } From 3e89e5ac6184f8eb9e68046fabafe72d32df282a Mon Sep 17 00:00:00 2001 From: niks3089 Date: Wed, 23 May 2018 18:27:03 +0530 Subject: [PATCH 346/723] ndp: Complete the wiring of ndp into the inet stack --- api/net/ip6/icmp6.hpp | 9 ++++++--- api/net/ip6/ip6.hpp | 8 ++------ api/net/ip6/ndp.hpp | 11 +++-------- src/net/inet.cpp | 9 +++++---- src/net/ip6/ip6.cpp | 15 ++------------- src/net/ip6/ndp.cpp | 14 ++++++++++---- 6 files changed, 28 insertions(+), 38 deletions(-) diff --git a/api/net/ip6/icmp6.hpp b/api/net/ip6/icmp6.hpp index b2c38b6c3a..d5e2117a3a 100644 --- a/api/net/ip6/icmp6.hpp +++ b/api/net/ip6/icmp6.hpp @@ -112,13 +112,16 @@ namespace net inline void set_network_out(downstream s) { network_layer_out_ = s; - // TODO: Remove this. NDP packets should be sent directly on link - ndp_.set_network_out(s); + } + + inline void set_ndp_linklayer_out(downstream_link s) + { + ndp_.set_linklayer_out(s); } void ndp_transmit(Packet_ptr ptr, IP6::addr next_hop) { - //ndp_.transmit(ptr, next_hop); + ndp_.transmit(std::move(ptr), next_hop); } /** diff --git a/api/net/ip6/ip6.hpp b/api/net/ip6/ip6.hpp index dbf650a6d5..e8d5a95564 100644 --- a/api/net/ip6/ip6.hpp +++ b/api/net/ip6/ip6.hpp @@ -83,13 +83,10 @@ namespace net void set_packet_forwarding(Forward_delg fwd) { forward_packet_ = fwd; } - /** Set linklayer out (downstream) */ - void set_ndp_out(downstream_ndp s) + /** Set linklayer out (downstream) via ndp */ + void set_linklayer_out(downstream_ndp s) { ndp_out_ = s; } - void set_linklayer_out(downstream_link link) - { linklayer_out_ = link; } - /** Upstream: Input from link layer */ void receive(Packet_ptr, const bool link_bcast); @@ -198,7 +195,6 @@ namespace net /** Downstream delegates */ downstream_ndp ndp_out_ = nullptr; - downstream_link linklayer_out_ = nullptr; /** Packet forwarding */ Forward_delg forward_packet_; diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp index 75e639df77..64922779f1 100644 --- a/api/net/ip6/ndp.hpp +++ b/api/net/ip6/ndp.hpp @@ -73,10 +73,6 @@ namespace net { void set_proxy_policy(Route_checker delg) { proxy_ = delg; } - /** Delegate link-layer output. */ - void set_linklayer_out(downstream_link link) - { linklayer_out_ = link; } - /** Downstream transmission. */ void transmit(Packet_ptr, IP6::addr next_hop); @@ -97,9 +93,9 @@ namespace net { flush_interval_ = m; } - // Delegate output to network layer - inline void set_network_out(downstream s) - { network_layer_out_ = s; }; + // Delegate output to link layer + void set_linklayer_out(downstream_link s) + { linklayer_out_ = s; }; private: @@ -162,7 +158,6 @@ namespace net { Stack& inet_; Route_checker proxy_ = nullptr; - downstream network_layer_out_ = nullptr; MAC::Addr mac_; diff --git a/src/net/inet.cpp b/src/net/inet.cpp index 6ed4923dd9..1dfc93879e 100644 --- a/src/net/inet.cpp +++ b/src/net/inet.cpp @@ -97,9 +97,11 @@ Inet::Inet(hw::Nic& nic) // IP4 -> Arp ip4_.set_linklayer_out(arp_top); - // IP6 -> Link - ip6_.set_linklayer_out(link_top); - //ip6_.set_linklayer_out(ndp_top); + // IP6 -> Ndp + ip6_.set_linklayer_out(ndp_top); + + // NDP -> Link + icmp6_.set_ndp_linklayer_out(link_top); // UDP6 -> IP6 // udp6->set_network_out(ip6_top); @@ -107,7 +109,6 @@ Inet::Inet(hw::Nic& nic) // tcp6->set_network_out(ip6_top); // Arp -> Link - // IP6 -> Link assert(link_top); arp_.set_linklayer_out(link_top); diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index 82fb98fcf4..fd374352fa 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -287,12 +287,7 @@ namespace net if (packet == nullptr) return; if (next_hop == IP6::ADDR_ANY) { -#if 0 - if (UNLIKELY(packet->ip_dst() == IP6::ADDR_BCAST)) { - next_hop = IP6::ADDR_BCAST; - } - else { -#endif + // Create local and target subnets addr target = packet->ip_dst() & stack_.netmask6(); addr local = stack_.ip6_addr() & stack_.netmask6(); @@ -318,13 +313,7 @@ namespace net // Stat increment packets transmitted packets_tx_++; - extern MAC::Addr linux_tap_device; - MAC::Addr dest_mac("c0:01:0a:00:00:01"); - - PRINT(" Transmitting packet on mac address: %s," - " layer begin: buf + %li\n", dest_mac.to_string().c_str(), - packet->layer_begin() - packet->buf()); - linklayer_out_(std::move(packet), dest_mac, Ethertype::IP6); + ndp_out_(std::move(packet), next_hop); } const ip6::Addr IP6::local_ip() const { diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index b622bb4868..1c54734631 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -81,7 +81,7 @@ namespace net PRINT(" Neighbor Adv Response size: %i payload size: %i, checksum: 0x%x\n", res.ip().size(), res.payload().size(), res.compute_checksum()); - network_layer_out_(res.release()); + transmit(res.release(), res.ip().ip_dst()); } void Ndp::receive_neighbor_advertisement(icmp6::Packet& req) @@ -168,7 +168,7 @@ namespace net PRINT(" Router solicit size: %i payload size: %i, checksum: 0x%x\n", req.ip().size(), req.payload().size(), req.compute_checksum()); - network_layer_out_(req.release()); + transmit(req.release(), req.ip().ip_dst()); } void Ndp::receive(icmp6::Packet& pckt) { @@ -301,8 +301,8 @@ namespace net PRINT(" physical> Transmitting %u bytes to %s\n", (uint32_t) pckt->size(), next_hop.str().c_str()); +#if 0 MAC::Addr dest_mac; - // If we don't have a cached IP, perform address resolution auto cache_entry = cache_.find(next_hop); if (UNLIKELY(cache_entry == cache_.end())) { @@ -316,9 +316,15 @@ namespace net PRINT(" Found cache entry for IP %s -> %s \n", next_hop.to_string().c_str(), dest_mac.to_string().c_str()); +#endif + MAC::Addr dest_mac("c0:01:0a:00:00:01"); + + PRINT("NDP: Transmitting packet on mac address: %s," + " layer begin: buf + %li\n", dest_mac.to_string().c_str(), + packet->layer_begin() - packet->buf()); // Move chain to linklayer - linklayer_out_(std::move(pckt), dest_mac, Ethertype::IP4); + linklayer_out_(std::move(pckt), dest_mac, Ethertype::IP6); } // NDP packet function definitions From 948297c4a57a9f6534305633f638223717a5c788 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Wed, 23 May 2018 19:39:08 +0530 Subject: [PATCH 347/723] ndp: remove hack in sending neighbor advertisement --- api/net/ip6/addr.hpp | 5 +++++ api/net/ip6/ndp.hpp | 5 ++++- src/net/ip6/ndp.cpp | 9 +++------ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index cb0c5a524b..0505c3dbf4 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -128,6 +128,11 @@ struct Addr { (ntohs(i32[3]) ^ 0xff)) == 0; } + uint8_t* data() + { + return reinterpret_cast (i16.data()); + } + /** * **/ diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp index 64922779f1..11922bd027 100644 --- a/api/net/ip6/ndp.hpp +++ b/api/net/ip6/ndp.hpp @@ -95,7 +95,10 @@ namespace net { // Delegate output to link layer void set_linklayer_out(downstream_link s) - { linklayer_out_ = s; }; + { linklayer_out_ = s; } + + MAC::Addr& link_mac_addr() + { return mac_; } private: diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index 1c54734631..29a898b7d4 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -68,12 +68,9 @@ namespace net res.ip().ip_dst().str().c_str()); // Insert target link address, ICMP6 option header and our mac address - // TODO: This is hacky. Fix it - MAC::Addr dest_mac("c0:01:0a:00:00:2a"); - // Target link address - res.set_payload({req.payload().data(), 16 }); - res.ndp().set_ndp_options_header(0x02, 0x01); - res.set_payload({reinterpret_cast (&dest_mac), 6}); + res.set_payload({req.ndp().neighbor_sol().get_target().data(), 16 }); + res.ndp().set_ndp_options_header(icmp6::ND_OPT_TARGET_LL_ADDR, 0x01); + res.set_payload({reinterpret_cast (&link_mac_addr()), 6}); // Add checksum res.set_checksum(); From 12659f8b9b75ea3aaf5c19c8a1466275f73c79ff Mon Sep 17 00:00:00 2001 From: niks3089 Date: Fri, 25 May 2018 21:50:46 +0530 Subject: [PATCH 348/723] ndp: Use ndp solicitation and advertisement to determine mac address --- api/net/ip6/addr.hpp | 28 ++++++ api/net/ip6/icmp6_common.hpp | 12 +-- api/net/ip6/ndp.hpp | 10 +- api/net/ip6/packet_icmp6.hpp | 19 +++- src/net/ip6/icmp6.cpp | 6 +- src/net/ip6/ndp.cpp | 186 ++++++++++++++++++++++++----------- 6 files changed, 182 insertions(+), 79 deletions(-) diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index 0505c3dbf4..a17087f7a1 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -133,6 +133,14 @@ struct Addr { return reinterpret_cast (i16.data()); } + Addr& solicit(const Addr other) noexcept { + i32[0] = htonl(0xFF020000); + i32[1] = 0; + i32[2] = htonl(0x1); + i32[3] = htonl(0xFF000000) | other.i32[3]; + return *this; + } + /** * **/ @@ -243,11 +251,31 @@ struct Addr { Addr operator~() const noexcept { return Addr{~i32[0], ~i32[1], ~i32[2], ~i32[3]}; } + uint8_t operator[](const int loc) const noexcept + { return i8[loc]; } + +#if 0 + template + T get_part(const uint8_t n) + { + constexpr if (T = uint8_t) { + Expects(n < 32) + return i8[n]; + } else if (T = uint16_t) { + Expects(n < 16) + return i16[n]; + } else { + static_assert(false, "Unallowed T"; + } + } +#endif + union { std::array i64; std::array i32; std::array i16; + std::array i8; }; } __attribute__((packed)); //< struct Addr } //< namespace ip6 diff --git a/api/net/ip6/icmp6_common.hpp b/api/net/ip6/icmp6_common.hpp index b5adda1078..2df05e83cd 100644 --- a/api/net/ip6/icmp6_common.hpp +++ b/api/net/ip6/icmp6_common.hpp @@ -35,10 +35,10 @@ namespace net { MULTICAST_LISTENER_QUERY = 130, MULTICAST_LISTENER_REPORT = 131, MULTICAST_LISTENER_DONE = 132, - ND_ROUTER_SOLICATION = 133, + ND_ROUTER_SOL = 133, ND_ROUTER_ADV = 134, - ND_NEIGHBOR_SOL = 135, - ND_NEIGHBOR_ADV = 136, + ND_NEIGHBOUR_SOL = 135, + ND_NEIGHBOUR_ADV = 136, ND_REDIRECT = 137, ROUTER_RENUMBERING = 138, INFORMATION_QUERY = 139, @@ -98,10 +98,10 @@ namespace net { case Type::MULTICAST_LISTENER_QUERY: case Type::MULTICAST_LISTENER_REPORT: case Type::MULTICAST_LISTENER_DONE: - case Type::ND_ROUTER_SOLICATION: + case Type::ND_ROUTER_SOL: case Type::ND_ROUTER_ADV: - case Type::ND_NEIGHBOR_SOL: - case Type::ND_NEIGHBOR_ADV: + case Type::ND_NEIGHBOUR_SOL: + case Type::ND_NEIGHBOUR_ADV: case Type::ND_REDIRECT: case Type::ROUTER_RENUMBERING: case Type::INFORMATION_QUERY: diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp index 11922bd027..22dc68b451 100644 --- a/api/net/ip6/ndp.hpp +++ b/api/net/ip6/ndp.hpp @@ -47,14 +47,14 @@ namespace net { /** Handle incoming NDP packet. */ void receive(icmp6::Packet& pckt); - void receive_neighbor_solicitation(icmp6::Packet& pckt); - void receive_neighbor_advertisement(icmp6::Packet& pckt); + void receive_neighbour_solicitation(icmp6::Packet& pckt); + void receive_neighbour_advertisement(icmp6::Packet& pckt); void receive_router_solicitation(icmp6::Packet& pckt); void receive_router_advertisement(icmp6::Packet& pckt); /** Send out NDP packet */ - void send_neighbor_solicitation(); - void send_neighbor_advertisement(icmp6::Packet& req); + void send_neighbour_solicitation(IP6::addr target); + void send_neighbour_advertisement(icmp6::Packet& req); void send_router_solicitation(); void send_router_advertisement(); @@ -74,7 +74,7 @@ namespace net { { proxy_ = delg; } /** Downstream transmission. */ - void transmit(Packet_ptr, IP6::addr next_hop); + void transmit(Packet_ptr, IP6::addr next_hop, MAC::Addr mac = MAC::EMPTY); /** Cache IP resolution. */ void cache(IP6::addr ip, MAC::Addr mac, uint8_t flags); diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index d32f16d22c..b4a2090b18 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -150,8 +150,8 @@ namespace icmp6 { IP6::addr target; uint8_t options[0]; - void set_target(IP6::addr tar) - { target = tar; } + IP6::addr get_target() + { return target; } uint16_t option_offset() { return 16; } @@ -177,13 +177,22 @@ namespace icmp6 { RouterRedirect& router_redirect() { return *reinterpret_cast(&(icmp6_.header().payload[0])); } - NeighborSol& neighbor_sol() + NeighborSol& neighbour_sol() { return *reinterpret_cast(&(icmp6_.header().payload[0])); } - NeighborAdv& neighbor_adv() + NeighborAdv& neighbour_adv() { return *reinterpret_cast(&(icmp6_.header().payload[0])); } - void set_neighbor_adv_flag(uint32_t flag) + bool is_flag_router() + { return icmp6_.header().rso_flags & NEIGH_ADV_ROUTER; } + + bool is_flag_solicited() + { return icmp6_.header().rso_flags & NEIGH_ADV_SOL; } + + bool is_flag_override() + { return icmp6_.header().rso_flags & NEIGH_ADV_OVERRIDE; } + + void set_neighbour_adv_flag(uint32_t flag) { icmp6_.header().rso_flags = htonl(flag << 28); } void set_ndp_options_header(uint8_t type, uint8_t len) diff --git a/src/net/ip6/icmp6.cpp b/src/net/ip6/icmp6.cpp index f529aefa14..e821879f3a 100644 --- a/src/net/ip6/icmp6.cpp +++ b/src/net/ip6/icmp6.cpp @@ -90,10 +90,10 @@ namespace net case (ICMP_type::MULTICAST_LISTENER_DONE): PRINT(" ICMP multicast message from %s\n", req.ip().ip_src().str().c_str()); break; - case (ICMP_type::ND_ROUTER_SOLICATION): + case (ICMP_type::ND_ROUTER_SOL): case (ICMP_type::ND_ROUTER_ADV): - case (ICMP_type::ND_NEIGHBOR_SOL): - case (ICMP_type::ND_NEIGHBOR_ADV): + case (ICMP_type::ND_NEIGHBOUR_SOL): + case (ICMP_type::ND_NEIGHBOUR_ADV): case (ICMP_type::ND_REDIRECT): ndp().receive(req); break; diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index 29a898b7d4..34fee4e3da 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -38,7 +38,7 @@ namespace net mac_ (inet.link_addr()) {} - void Ndp::send_neighbor_advertisement(icmp6::Packet& req) + void Ndp::send_neighbour_advertisement(icmp6::Packet& req) { icmp6::Packet res(inet_.ip6_packet_factory()); bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; @@ -61,38 +61,98 @@ namespace net res.ip().set_ip_hop_limit(255); // Populate response ICMP header - res.set_type(ICMP_type::ND_NEIGHBOR_ADV); + res.set_type(ICMP_type::ND_NEIGHBOUR_ADV); res.set_code(0); - res.ndp().set_neighbor_adv_flag(NEIGH_ADV_SOL | NEIGH_ADV_OVERRIDE); - PRINT(" Transmitting Neighbor adv to %s\n", + res.ndp().set_neighbour_adv_flag(NEIGH_ADV_SOL | NEIGH_ADV_OVERRIDE); + PRINT("NDP: Transmitting Neighbor adv to %s\n", res.ip().ip_dst().str().c_str()); // Insert target link address, ICMP6 option header and our mac address - res.set_payload({req.ndp().neighbor_sol().get_target().data(), 16 }); + res.set_payload({req.ndp().neighbour_sol().get_target().data(), 16 }); res.ndp().set_ndp_options_header(icmp6::ND_OPT_TARGET_LL_ADDR, 0x01); res.set_payload({reinterpret_cast (&link_mac_addr()), 6}); // Add checksum res.set_checksum(); - PRINT(" Neighbor Adv Response size: %i payload size: %i, checksum: 0x%x\n", + PRINT("NDP: Neighbor Adv Response size: %i payload size: %i, checksum: 0x%x\n", res.ip().size(), res.payload().size(), res.compute_checksum()); - transmit(res.release(), res.ip().ip_dst()); + auto dest = res.ip().ip_dst(); + transmit(res.release(), dest); } - void Ndp::receive_neighbor_advertisement(icmp6::Packet& req) + void Ndp::receive_neighbour_advertisement(icmp6::Packet& req) { + IP6::addr target = req.ndp().neighbour_adv().get_target(); + uint8_t *lladdr; + + if (target.is_multicast()) { + PRINT("NDP: neighbour advertisement target address is multicast\n"); + return; + } + + if (req.ip().ip_dst().is_multicast() && + req.ndp().is_flag_solicited()) { + PRINT("NDP: neighbour destination address is multicast when" + " solicit flag is set\n"); + return; + } + + req.ndp().parse(ICMP_type::ND_NEIGHBOUR_ADV); + lladdr = req.ndp().get_option_data(icmp6::ND_OPT_TARGET_LL_ADDR); + + // For now, just create a cache entry, if one doesn't exist + if (lookup(true, target, lladdr, 0) == false) { + PRINT("NDP: Failed to create a cached entry for %s\n", + target.str().c_str()); + } else { + auto waiting = waiting_packets_.find(target); + if (waiting != waiting_packets_.end()) { + PRINT("Ndp: Had a packet waiting for this IP. Sending\n"); + transmit(std::move(waiting->second.pckt), target); + waiting_packets_.erase(waiting); + } + } } - void Ndp::send_neighbor_solicitation() + void Ndp::send_neighbour_solicitation(IP6::addr target) { + IP6::addr dest_ip; + icmp6::Packet req(inet_.ip6_packet_factory()); + + req.ip().set_ip_src(inet_.ip6_addr()); + req.ip().set_ip_dst(dest_ip.solicit(target)); + + req.ip().set_ip_hop_limit(255); + req.set_type(ICMP_type::ND_NEIGHBOUR_SOL); + req.set_code(0); + req.set_reserved(0); + + // Set target address + req.set_payload({target.data(), 16}); + req.ndp().set_ndp_options_header(icmp6::ND_OPT_SOURCE_LL_ADDR, 0x01); + req.set_payload({reinterpret_cast (&link_mac_addr()), 6}); + + req.set_checksum(); + + MAC::Addr dest_mac(0x33,0x33, req.ip().ip_dst()[12], + req.ip().ip_dst()[13], req.ip().ip_dst()[14], + req.ip().ip_dst()[15]); + PRINT("NDP: Sending Neighbour solicit size: %i payload size: %i," + "checksum: 0x%x\n, source: %s, dest: %s, dest mac: %s\n", + req.ip().size(), req.payload().size(), req.compute_checksum(), + req.ip().ip_src().str().c_str(), + req.ip().ip_dst().str().c_str(), dest_mac.str().c_str()); + + auto dest = req.ip().ip_dst(); + transmit(req.release(), dest, dest_mac); } - void Ndp::receive_neighbor_solicitation(icmp6::Packet& req) + void Ndp::receive_neighbour_solicitation(icmp6::Packet& req) { bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; - IP6::addr target = req.ndp().neighbor_sol().get_target(); + IP6::addr target = req.ndp().neighbour_sol().get_target(); uint8_t *lladdr, *nonce_opt; bool found; uint64_t nonce = 0; @@ -101,21 +161,21 @@ namespace net PRINT("target: %s\n", target.str().c_str()); if (target.is_multicast()) { - PRINT("ND: neighbor solictation target address is multicast\n"); + PRINT("NDP: neighbour solictation target address is multicast\n"); return; } if (any_src && !req.ip().ip_dst().is_solicit_multicast()) { - PRINT("ND: neighbor solictation address is any source " + PRINT("NDP: neighbour solictation address is any source " "but not solicit destination\n"); return; } - req.ndp().parse(ICMP_type::ND_NEIGHBOR_SOL); + req.ndp().parse(ICMP_type::ND_NEIGHBOUR_SOL); lladdr = req.ndp().get_option_data(icmp6::ND_OPT_SOURCE_LL_ADDR); if (lladdr) { if (any_src) { - PRINT("ND: bad any source packet with link layer option\n"); + PRINT("NDP: bad any source packet with link layer option\n"); return; } } @@ -133,7 +193,7 @@ namespace net } if (any_src) { - send_neighbor_advertisement(req); + send_neighbour_advertisement(req); return; } @@ -142,7 +202,7 @@ namespace net req.ip().ip_src(), lladdr, 0); if (found) { - send_neighbor_advertisement(req); + send_neighbour_advertisement(req); } } @@ -153,7 +213,7 @@ namespace net req.ip().set_ip_src(inet_.ip6_addr()); req.ip().set_ip_dst(ip6::Addr::node_all_nodes); req.ip().set_ip_hop_limit(255); - req.set_type(ICMP_type::ND_ROUTER_SOLICATION); + req.set_type(ICMP_type::ND_ROUTER_SOL); req.set_code(0); req.set_reserved(0); // Set multicast addreqs @@ -162,36 +222,47 @@ namespace net // Add checksum req.set_checksum(); - PRINT(" Router solicit size: %i payload size: %i, checksum: 0x%x\n", + PRINT("NDP: Router solicit size: %i payload size: %i, checksum: 0x%x\n", req.ip().size(), req.payload().size(), req.compute_checksum()); transmit(req.release(), req.ip().ip_dst()); } + void Ndp::receive_router_solicitation(icmp6::Packet& req) + { + } + + void Ndp::receive_router_advertisement(icmp6::Packet& req) + { + } + void Ndp::receive(icmp6::Packet& pckt) { switch(pckt.type()) { - case (ICMP_type::ND_ROUTER_SOLICATION): - PRINT(" ICMP Router solictation message from %s\n", pckt.ip().ip_src().str().c_str()); + case (ICMP_type::ND_ROUTER_SOL): + PRINT("NDP: Router solictation message from %s\n", pckt.ip().ip_src().str().c_str()); + receive_router_solicitation(pckt); break; case (ICMP_type::ND_ROUTER_ADV): - PRINT(" ICMP Router advertisement message from %s\n", pckt.ip().ip_src().str().c_str()); + PRINT("NDP: Router advertisement message from %s\n", pckt.ip().ip_src().str().c_str()); + receive_router_advertisement(pckt); break; - case (ICMP_type::ND_NEIGHBOR_SOL): - PRINT(" ICMP Neigbor solictation message from %s\n", pckt.ip().ip_src().str().c_str()); - receive_neighbor_solicitation(pckt); + case (ICMP_type::ND_NEIGHBOUR_SOL): + PRINT("NDP: Neigbor solictation message from %s\n", pckt.ip().ip_src().str().c_str()); + receive_neighbour_solicitation(pckt); break; - case (ICMP_type::ND_NEIGHBOR_ADV): - PRINT(" ICMP Neigbor advertisement message from %s\n", pckt.ip().ip_src().str().c_str()); + case (ICMP_type::ND_NEIGHBOUR_ADV): + PRINT("NDP: Neigbor advertisement message from %s\n", pckt.ip().ip_src().str().c_str()); + receive_neighbour_advertisement(pckt); break; case (ICMP_type::ND_REDIRECT): - PRINT(" ICMP Neigbor redirect message from %s\n", pckt.ip().ip_src().str().c_str()); + PRINT("NDP: Neigbor redirect message from %s\n", pckt.ip().ip_src().str().c_str()); break; default: return; } } - bool Ndp::lookup(bool create, IP6::addr ip, uint8_t *ll_addr, uint8_t flags) + bool Ndp::lookup(bool create, IP6::addr ip, uint8_t *ll_addr, uint8_t flags = 0) { auto entry = cache_.find(ip); if (entry != cache_.end()) { @@ -263,7 +334,7 @@ namespace net void Ndp::flush_expired() { - PRINT(" Flushing expired entries\n"); + PRINT("NDP: Flushing expired entries\n"); std::vector expired; for (auto ent : cache_) { if (ent.second.expired()) { @@ -288,40 +359,35 @@ namespace net requests_tx_++; // Send ndp solicit + send_neighbour_solicitation(next_hop); } - void Ndp::transmit(Packet_ptr pckt, IP6::addr next_hop) + void Ndp::transmit(Packet_ptr pckt, IP6::addr next_hop, MAC::Addr mac) { Expects(pckt->size()); - PRINT(" physical> Transmitting %u bytes to %s\n", - (uint32_t) pckt->size(), next_hop.str().c_str()); - -#if 0 - MAC::Addr dest_mac; - // If we don't have a cached IP, perform address resolution - auto cache_entry = cache_.find(next_hop); - if (UNLIKELY(cache_entry == cache_.end())) { - PRINT(" No cache entry for IP %s. Resolving. \n", next_hop.to_string().c_str()); - await_resolution(std::move(pckt), next_hop); - return; - } + if (mac == MAC::EMPTY) { + // If we don't have a cached IP, perform NDP sol + auto cache_entry = cache_.find(next_hop); + if (UNLIKELY(cache_entry == cache_.end())) { + PRINT("NDP: No cache entry for IP %s. Resolving. \n", next_hop.to_string().c_str()); + await_resolution(std::move(pckt), next_hop); + return; + } - // Get MAC from cache - dest_mac = cache_[next_hop].mac(); + // Get MAC from cache + mac = cache_[next_hop].mac(); - PRINT(" Found cache entry for IP %s -> %s \n", - next_hop.to_string().c_str(), dest_mac.to_string().c_str()); -#endif - MAC::Addr dest_mac("c0:01:0a:00:00:01"); + PRINT("NDP: Found cache entry for IP %s -> %s \n", + next_hop.to_string().c_str(), mac.to_string().c_str()); + } - PRINT("NDP: Transmitting packet on mac address: %s," - " layer begin: buf + %li\n", dest_mac.to_string().c_str(), - packet->layer_begin() - packet->buf()); + PRINT(" physical> Transmitting %u bytes to %s\n", + (uint32_t) pckt->size(), next_hop.str().c_str()); // Move chain to linklayer - linklayer_out_(std::move(pckt), dest_mac, Ethertype::IP6); + linklayer_out_(std::move(pckt), mac, Ethertype::IP6); } // NDP packet function definitions @@ -329,19 +395,19 @@ namespace net void Packet::NdpPacket::parse(icmp6::Type type) { switch(type) { - case (ICMP_type::ND_ROUTER_SOLICATION): + case (ICMP_type::ND_ROUTER_SOL): ndp_opt_.parse(router_sol().options, (icmp6_.payload_len() - router_sol().option_offset())); break; case (ICMP_type::ND_ROUTER_ADV): break; - case (ICMP_type::ND_NEIGHBOR_SOL): - ndp_opt_.parse(neighbor_sol().options, - (icmp6_.payload_len() - neighbor_sol().option_offset())); + case (ICMP_type::ND_NEIGHBOUR_SOL): + ndp_opt_.parse(neighbour_sol().options, + (icmp6_.payload_len() - neighbour_sol().option_offset())); break; - case (ICMP_type::ND_NEIGHBOR_ADV): - ndp_opt_.parse(neighbor_adv().options, - (icmp6_.payload_len() - neighbor_adv().option_offset())); + case (ICMP_type::ND_NEIGHBOUR_ADV): + ndp_opt_.parse(neighbour_adv().options, + (icmp6_.payload_len() - neighbour_adv().option_offset())); break; case (ICMP_type::ND_REDIRECT): ndp_opt_.parse(router_redirect().options, From d3700b9e8854f0a4171d9240ff3147a7659df448 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Sun, 27 May 2018 16:47:51 +0530 Subject: [PATCH 349/723] ndp: New cache apis and provide support for ndp flags --- api/hw/mac_addr.hpp | 4 +- api/net/ip6/addr.hpp | 3 -- api/net/ip6/ndp.hpp | 73 +++++++++++++++++++++++++++++++----- src/net/ip6/ndp.cpp | 89 ++++++++++++++++++++++++++------------------ 4 files changed, 118 insertions(+), 51 deletions(-) diff --git a/api/hw/mac_addr.hpp b/api/hw/mac_addr.hpp index 17a6c94893..b07b8eaa50 100644 --- a/api/hw/mac_addr.hpp +++ b/api/hw/mac_addr.hpp @@ -89,7 +89,9 @@ union Addr { Addr(uint8_t *addr) noexcept { - memcpy(part, addr, PARTS_LEN); + if (addr) { + memcpy(part, addr, PARTS_LEN); + } } /** diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index a17087f7a1..1711e45ed0 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -261,9 +261,6 @@ struct Addr { constexpr if (T = uint8_t) { Expects(n < 32) return i8[n]; - } else if (T = uint16_t) { - Expects(n < 16) - return i16[n]; } else { static_assert(false, "Unallowed T"; } diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp index 22dc68b451..8d6cf3c305 100644 --- a/api/net/ip6/ndp.hpp +++ b/api/net/ip6/ndp.hpp @@ -34,6 +34,21 @@ namespace net { class Ndp { public: + +#define NEIGH_UPDATE_OVERRIDE 0x00000001 +#define NEIGH_UPDATE_WEAK_OVERRIDE 0x00000002 +#define NEIGH_UPDATE_OVERRIDE_ISROUTER 0x00000004 +#define NEIGH_UPDATE_ISROUTER 0x40000000 +#define NEIGH_UPDATE_ADMIN 0x80000000 + + enum class NeighbourStates : uint8_t { + INCOMPLETE, + REACHABLE, + STALE, + DELAY, + PROBE, + MAX + }; using Stack = IP6::Stack; using Route_checker = delegate; using Ndp_resolver = delegate; @@ -77,19 +92,20 @@ namespace net { void transmit(Packet_ptr, IP6::addr next_hop, MAC::Addr mac = MAC::EMPTY); /** Cache IP resolution. */ - void cache(IP6::addr ip, MAC::Addr mac, uint8_t flags); + void cache(IP6::addr ip, MAC::Addr mac, NeighbourStates state, uint32_t flags); + void cache(IP6::addr ip, uint8_t *ll_addr, NeighbourStates state, uint32_t flags); /** Lookup for cache entry */ - bool lookup(bool create, IP6::addr ip, uint8_t *ll_addr, uint8_t flags); + bool lookup(IP6::addr ip, uint8_t *ll_addr); /** Flush the NDP cache. RFC-2.3.2.1 */ void flush_cache() - { cache_.clear(); }; + { neighbour_cache_.clear(); }; /** Flush expired cache entries. RFC-2.3.2.1 */ void flush_expired (); - void set_cache_flush_interval(std::chrono::minutes m) { + void set_neighbour_cache_flush_interval(std::chrono::minutes m) { flush_interval_ = m; } @@ -102,8 +118,8 @@ namespace net { private: - /** NDP cache expires after cache_exp_sec_ seconds */ - static constexpr uint16_t cache_exp_sec_ {60 * 5}; + /** NDP cache expires after neighbour_cache_exp_sec_ seconds */ + static constexpr uint16_t neighbour_cache_exp_sec_ {60 * 5}; /** Cache entries are just MAC's and timestamps */ struct Cache_entry { @@ -113,13 +129,18 @@ namespace net { Cache_entry(MAC::Addr mac) noexcept : mac_(mac), timestamp_(RTC::time_since_boot()) {} + Cache_entry(MAC::Addr mac, NeighbourStates state, uint32_t flags) noexcept + : mac_(mac), timestamp_(RTC::time_since_boot()), flags_(flags) { + set_state(state); + } + Cache_entry(const Cache_entry& cpy) noexcept : mac_(cpy.mac_), timestamp_(cpy.timestamp_) {} void update() noexcept { timestamp_ = RTC::time_since_boot(); } bool expired() const noexcept - { return RTC::time_since_boot() > timestamp_ + cache_exp_sec_; } + { return RTC::time_since_boot() > timestamp_ + neighbour_cache_exp_sec_; } MAC::Addr mac() const noexcept { return mac_; } @@ -128,11 +149,43 @@ namespace net { { return timestamp_; } RTC::timestamp_t expires() const noexcept - { return timestamp_ + cache_exp_sec_; } + { return timestamp_ + neighbour_cache_exp_sec_; } + + void set_state(NeighbourStates state) + { + if (state < NeighbourStates::MAX) { + state_ = state; + } + } + + void set_flags(uint32_t flags) + { + flags_ = flags; + } + + std::string get_state_name() + { + switch(state_) { + case NeighbourStates::INCOMPLETE: + return "incomplete"; + case NeighbourStates::REACHABLE: + return "reachable"; + case NeighbourStates::STALE: + return "stale"; + case NeighbourStates::DELAY: + return "delay"; + case NeighbourStates::PROBE: + return "probe"; + default: + return "uknown"; + } + } private: - MAC::Addr mac_; + uint32_t flags_; + MAC::Addr mac_; RTC::timestamp_t timestamp_; + NeighbourStates state_; }; //< struct Cache_entry struct Queue_entry { @@ -168,7 +221,7 @@ namespace net { downstream_link linklayer_out_ = nullptr; // The NDP cache - Cache cache_ {}; + Cache neighbour_cache_ {}; // RFC-1122 2.3.2.2 Packet queue PacketQueue waiting_packets_; diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index 34fee4e3da..26973e00e4 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -103,16 +103,18 @@ namespace net lladdr = req.ndp().get_option_data(icmp6::ND_OPT_TARGET_LL_ADDR); // For now, just create a cache entry, if one doesn't exist - if (lookup(true, target, lladdr, 0) == false) { - PRINT("NDP: Failed to create a cached entry for %s\n", - target.str().c_str()); - } else { - auto waiting = waiting_packets_.find(target); - if (waiting != waiting_packets_.end()) { - PRINT("Ndp: Had a packet waiting for this IP. Sending\n"); - transmit(std::move(waiting->second.pckt), target); - waiting_packets_.erase(waiting); - } + cache(target, lladdr, req.ndp().is_flag_solicited() ? + NeighbourStates::REACHABLE : NeighbourStates::STALE, + NEIGH_UPDATE_WEAK_OVERRIDE | + (req.ndp().is_flag_override() ? NEIGH_UPDATE_OVERRIDE : 0) | + NEIGH_UPDATE_OVERRIDE_ISROUTER | + (req.ndp().is_flag_router() ? NEIGH_UPDATE_ISROUTER : 0)); + + auto waiting = waiting_packets_.find(target); + if (waiting != waiting_packets_.end()) { + PRINT("Ndp: Had a packet waiting for this IP. Sending\n"); + transmit(std::move(waiting->second.pckt), target); + waiting_packets_.erase(waiting); } } @@ -122,13 +124,16 @@ namespace net icmp6::Packet req(inet_.ip6_packet_factory()); req.ip().set_ip_src(inet_.ip6_addr()); - req.ip().set_ip_dst(dest_ip.solicit(target)); req.ip().set_ip_hop_limit(255); req.set_type(ICMP_type::ND_NEIGHBOUR_SOL); req.set_code(0); req.set_reserved(0); + // Solicit destination address. Source address + // option must be present + req.ip().set_ip_dst(dest_ip.solicit(target)); + // Set target address req.set_payload({target.data(), 16}); req.ndp().set_ndp_options_header(icmp6::ND_OPT_SOURCE_LL_ADDR, 0x01); @@ -146,6 +151,7 @@ namespace net req.ip().ip_dst().str().c_str(), dest_mac.str().c_str()); auto dest = req.ip().ip_dst(); + cache(dest, MAC::EMPTY, NeighbourStates::INCOMPLETE, 0); transmit(req.release(), dest, dest_mac); } @@ -193,18 +199,17 @@ namespace net } if (any_src) { - send_neighbour_advertisement(req); + if (lladdr) { + send_neighbour_advertisement(req); + } return; } /* Update/Create cache entry for the source address */ - found = lookup(!is_dest_multicast || lladdr, - req.ip().ip_src(), lladdr, 0); - - if (found) { - send_neighbour_advertisement(req); - } + cache(req.ip().ip_src(), lladdr, NeighbourStates::STALE, + NEIGH_UPDATE_WEAK_OVERRIDE| NEIGH_UPDATE_OVERRIDE); + send_neighbour_advertisement(req); } void Ndp::send_router_solicitation() @@ -262,34 +267,42 @@ namespace net } } - bool Ndp::lookup(bool create, IP6::addr ip, uint8_t *ll_addr, uint8_t flags = 0) + bool Ndp::lookup(IP6::addr ip, uint8_t *ll_addr) { - auto entry = cache_.find(ip); - if (entry != cache_.end()) { - return true; - } else if (create == true && ll_addr) { - MAC::Addr mac(ll_addr); - cache(ip, mac, flags); + auto entry = neighbour_cache_.find(ip); + if (entry != neighbour_cache_.end()) { return true; } return false; } - void Ndp::cache(IP6::addr ip, MAC::Addr mac, uint8_t flags) + void Ndp::cache(IP6::addr ip, uint8_t *ll_addr, NeighbourStates state, uint32_t flags) + { + if (ll_addr) { + MAC::Addr mac(ll_addr); + cache(ip, mac, state, flags); + } + } + + void Ndp::cache(IP6::addr ip, MAC::Addr mac, NeighbourStates state, uint32_t flags) { PRINT("Ndp Caching IP %s for %s\n", ip.str().c_str(), mac.str().c_str()); - auto entry = cache_.find(ip); - if (entry != cache_.end()) { + auto entry = neighbour_cache_.find(ip); + if (entry != neighbour_cache_.end()) { PRINT("Cached entry found: %s recorded @ %zu. Updating timestamp\n", entry->second.mac().str().c_str(), entry->second.timestamp()); if (entry->second.mac() != mac) { - cache_.erase(entry); - cache_.emplace(ip, mac); + neighbour_cache_.erase(entry); + neighbour_cache_.emplace( + std::make_pair(ip, Cache_entry{mac, state, flags})); // Insert } else { + entry->second.set_state(state); + entry->second.set_flags(flags); entry->second.update(); } } else { - cache_.emplace(ip, mac); // Insert + neighbour_cache_.emplace( + std::make_pair(ip, Cache_entry{mac, state, flags})); // Insert if (UNLIKELY(not flush_timer_.is_running())) { flush_timer_.start(flush_interval_); } @@ -305,6 +318,8 @@ namespace net ndp_resolver_(it->first); it++; } else { + // TODO: According to RFC, + // Send ICMP destination unreachable it = waiting_packets_.erase(it); } } @@ -336,17 +351,17 @@ namespace net { PRINT("NDP: Flushing expired entries\n"); std::vector expired; - for (auto ent : cache_) { + for (auto ent : neighbour_cache_) { if (ent.second.expired()) { expired.push_back(ent.first); } } for (auto ip : expired) { - cache_.erase(ip); + neighbour_cache_.erase(ip); } - if (not cache_.empty()) { + if (not neighbour_cache_.empty()) { flush_timer_.start(flush_interval_); } } @@ -369,15 +384,15 @@ namespace net if (mac == MAC::EMPTY) { // If we don't have a cached IP, perform NDP sol - auto cache_entry = cache_.find(next_hop); - if (UNLIKELY(cache_entry == cache_.end())) { + auto neighbour_cache_entry = neighbour_cache_.find(next_hop); + if (UNLIKELY(neighbour_cache_entry == neighbour_cache_.end())) { PRINT("NDP: No cache entry for IP %s. Resolving. \n", next_hop.to_string().c_str()); await_resolution(std::move(pckt), next_hop); return; } // Get MAC from cache - mac = cache_[next_hop].mac(); + mac = neighbour_cache_[next_hop].mac(); PRINT("NDP: Found cache entry for IP %s -> %s \n", next_hop.to_string().c_str(), mac.to_string().c_str()); From 9fb90cf4b667fe98aacee353f016a538853641d5 Mon Sep 17 00:00:00 2001 From: Syeda Taiyeba Haroon Date: Tue, 29 May 2018 13:29:34 +0200 Subject: [PATCH 350/723] Misc : running build examples in parallel --- test/misc/build_examples/test.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/misc/build_examples/test.sh b/test/misc/build_examples/test.sh index ef4b01e73e..3e9cef0100 100755 --- a/test/misc/build_examples/test.sh +++ b/test/misc/build_examples/test.sh @@ -54,12 +54,15 @@ function build_service() { echo "[ PASS ]" } +export -f build_service + for dir in `ls -d $script_absolute_dir/../../../examples/*` do if [[ $dir == *"$skip_tests"* ]]; then continue fi - build_service "$dir" + # build_service "$dir" + parallel build_service ::: "$dir" done echo "Done" From df2d49ec86fefd740b1a4ba1f09ac06de2ff6e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Tue, 29 May 2018 13:32:36 +0200 Subject: [PATCH 351/723] cmake: Add libos.a between TLS and ext libraries --- cmake/post.service.cmake | 1 + test/kernel/integration/grub/setup.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 7ab0ce4d95..bad2632b0f 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -451,6 +451,7 @@ endif() target_link_libraries(service libos ${LIBR_CMAKE_NAMES} + libos libbotan ${OPENSSL_LIBS} libosdeps diff --git a/test/kernel/integration/grub/setup.sh b/test/kernel/integration/grub/setup.sh index 3a0f4c1c51..d9d21692de 100755 --- a/test/kernel/integration/grub/setup.sh +++ b/test/kernel/integration/grub/setup.sh @@ -1,2 +1,2 @@ #!/bin/bash -sudo apt install -y grub-pc xorriso +sudo apt install -qy grub-pc xorriso From 8e48774ef147106d43cb9589920576d4a58b4d7d Mon Sep 17 00:00:00 2001 From: niks3089 Date: Tue, 29 May 2018 17:35:11 +0530 Subject: [PATCH 352/723] ndp: Use get_part instead of [] operator to retrieve the address --- api/net/ip6/addr.hpp | 36 +++++++++++++++++++----------------- src/net/ip6/ndp.cpp | 8 +++++--- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index 1711e45ed0..be8f6ce846 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -148,6 +148,25 @@ struct Addr { { return i32[0] == 0 && i32[1] == 0 && i32[2] == 0 && ntohl(i32[3]) == 1; } + template + T get_part(const uint8_t n) const + { + static_assert(std::is_same_v or + std::is_same_v or + std::is_same_v, "Unallowed T"); + + if constexpr (std::is_same_v) { + Expects(n < 16); + return i8[n]; + } else if constexpr (std::is_same_v) { + Expects(n < 8); + return i16[n]; + } else if constexpr (std::is_same_v) { + Expects(n < 4); + return i32[n]; + } + } + /** * Assignment operator */ @@ -185,7 +204,6 @@ struct Addr { return false; } - /** * Operator to check for greater-than-or-equal relationship */ @@ -251,22 +269,6 @@ struct Addr { Addr operator~() const noexcept { return Addr{~i32[0], ~i32[1], ~i32[2], ~i32[3]}; } - uint8_t operator[](const int loc) const noexcept - { return i8[loc]; } - -#if 0 - template - T get_part(const uint8_t n) - { - constexpr if (T = uint8_t) { - Expects(n < 32) - return i8[n]; - } else { - static_assert(false, "Unallowed T"; - } - } -#endif - union { std::array i64; diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index 26973e00e4..45a1ff6f58 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -141,9 +141,11 @@ namespace net req.set_checksum(); - MAC::Addr dest_mac(0x33,0x33, req.ip().ip_dst()[12], - req.ip().ip_dst()[13], req.ip().ip_dst()[14], - req.ip().ip_dst()[15]); + MAC::Addr dest_mac(0x33,0x33, + req.ip().ip_dst().get_part(12), + req.ip().ip_dst().get_part(13), + req.ip().ip_dst().get_part(14), + req.ip().ip_dst().get_part(15)); PRINT("NDP: Sending Neighbour solicit size: %i payload size: %i," "checksum: 0x%x\n, source: %s, dest: %s, dest mac: %s\n", req.ip().size(), req.payload().size(), req.compute_checksum(), From 59b635be214ad7765ad92f5542bb4c23e0efab4e Mon Sep 17 00:00:00 2001 From: niks3089 Date: Tue, 29 May 2018 18:17:46 +0530 Subject: [PATCH 353/723] ndp: Minor nits, cleanup --- api/net/ip6/header.hpp | 1 + api/net/ip6/icmp6_common.hpp | 4 ++-- api/net/ip6/ndp.hpp | 24 +++++++++--------------- api/net/ip6/packet_icmp6.hpp | 31 +++++++++++++++---------------- src/net/ip6/ip6.cpp | 1 - src/net/ip6/ndp.cpp | 14 ++++++++------ 6 files changed, 35 insertions(+), 40 deletions(-) diff --git a/api/net/ip6/header.hpp b/api/net/ip6/header.hpp index 69f37b0965..21160127b4 100644 --- a/api/net/ip6/header.hpp +++ b/api/net/ip6/header.hpp @@ -22,6 +22,7 @@ #include #include #define IP6_HEADER_LEN 40 +#define IP6_ADDR_BYTES 16 namespace net { namespace ip6 { diff --git a/api/net/ip6/icmp6_common.hpp b/api/net/ip6/icmp6_common.hpp index 2df05e83cd..59f5323854 100644 --- a/api/net/ip6/icmp6_common.hpp +++ b/api/net/ip6/icmp6_common.hpp @@ -37,8 +37,8 @@ namespace net { MULTICAST_LISTENER_DONE = 132, ND_ROUTER_SOL = 133, ND_ROUTER_ADV = 134, - ND_NEIGHBOUR_SOL = 135, - ND_NEIGHBOUR_ADV = 136, + ND_NEIGHBOUR_SOL = 135, + ND_NEIGHBOUR_ADV = 136, ND_REDIRECT = 137, ROUTER_RENUMBERING = 138, INFORMATION_QUERY = 139, diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp index 8d6cf3c305..456acebcd6 100644 --- a/api/net/ip6/ndp.hpp +++ b/api/net/ip6/ndp.hpp @@ -47,7 +47,7 @@ namespace net { STALE, DELAY, PROBE, - MAX + FAIL }; using Stack = IP6::Stack; using Route_checker = delegate; @@ -98,11 +98,11 @@ namespace net { /** Lookup for cache entry */ bool lookup(IP6::addr ip, uint8_t *ll_addr); - /** Flush the NDP cache. RFC-2.3.2.1 */ + /** Flush the NDP cache */ void flush_cache() { neighbour_cache_.clear(); }; - /** Flush expired cache entries. RFC-2.3.2.1 */ + /** Flush expired cache entries */ void flush_expired (); void set_neighbour_cache_flush_interval(std::chrono::minutes m) { @@ -126,16 +126,14 @@ namespace net { /** Map needs empty constructor (we have no emplace yet) */ Cache_entry() noexcept = default; - Cache_entry(MAC::Addr mac) noexcept - : mac_(mac), timestamp_(RTC::time_since_boot()) {} - Cache_entry(MAC::Addr mac, NeighbourStates state, uint32_t flags) noexcept : mac_(mac), timestamp_(RTC::time_since_boot()), flags_(flags) { set_state(state); } Cache_entry(const Cache_entry& cpy) noexcept - : mac_(cpy.mac_), timestamp_(cpy.timestamp_) {} + : mac_(cpy.mac_), state_(cpy.state_), + timestamp_(cpy.timestamp_), flags_(cpy.flags_) {} void update() noexcept { timestamp_ = RTC::time_since_boot(); } @@ -153,7 +151,7 @@ namespace net { void set_state(NeighbourStates state) { - if (state < NeighbourStates::MAX) { + if (state <= NeighbourStates::FAIL) { state_ = state; } } @@ -176,6 +174,8 @@ namespace net { return "delay"; case NeighbourStates::PROBE: return "probe"; + case NeighbourStates::FAIL: + return "fail"; default: return "uknown"; } @@ -200,7 +200,6 @@ namespace net { using Cache = std::unordered_map; using PacketQueue = std::unordered_map; - /** Stats */ uint32_t& requests_rx_; uint32_t& requests_tx_; @@ -223,7 +222,7 @@ namespace net { // The NDP cache Cache neighbour_cache_ {}; - // RFC-1122 2.3.2.2 Packet queue + // Packet queue PacketQueue waiting_packets_; // Settable resolver - defualts to ndp_resolve @@ -234,10 +233,6 @@ namespace net { /** * Add a packet to waiting queue, to be sent when IP is resolved. - * - * Implements RFC1122 - * 2.3.2.1 : Prevent NDP flooding - * 2.3.2.2 : Packets SHOULD be queued. */ void await_resolution(Packet_ptr, IP6::addr); @@ -247,7 +242,6 @@ namespace net { /** Retry ndp-resolution for packets still waiting */ void resolve_waiting(); - }; //< class Ndp } //< namespace net diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index b4a2090b18..4b0e8b7ad3 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -36,17 +36,17 @@ namespace icmp6 { enum { ND_OPT_PREFIX_INFO_END = 0, - ND_OPT_SOURCE_LL_ADDR = 1, /* RFC2461 */ - ND_OPT_TARGET_LL_ADDR = 2, /* RFC2461 */ - ND_OPT_PREFIX_INFO = 3, /* RFC2461 */ + ND_OPT_SOURCE_LL_ADDR = 1, /* RFC2461 */ + ND_OPT_TARGET_LL_ADDR = 2, /* RFC2461 */ + ND_OPT_PREFIX_INFO = 3, /* RFC2461 */ ND_OPT_REDIRECT_HDR = 4, /* RFC2461 */ - ND_OPT_MTU = 5, /* RFC2461 */ - ND_OPT_NONCE = 14, /* RFC7527 */ + ND_OPT_MTU = 5, /* RFC2461 */ + ND_OPT_NONCE = 14, /* RFC7527 */ ND_OPT_ARRAY_MAX, - ND_OPT_ROUTE_INFO = 24, /* RFC4191 */ - ND_OPT_RDNSS = 25, /* RFC5006 */ - ND_OPT_DNSSL = 31, /* RFC6106 */ - ND_OPT_6CO = 34, /* RFC6775 */ + ND_OPT_ROUTE_INFO = 24, /* RFC4191 */ + ND_OPT_RDNSS = 25, /* RFC5006 */ + ND_OPT_DNSSL = 31, /* RFC6106 */ + ND_OPT_6CO = 34, /* RFC6775 */ ND_OPT_MAX }; @@ -66,11 +66,11 @@ namespace icmp6 { private: struct nd_options_header *header_; - std::array opt_array; struct nd_options_header *nd_opts_ri; struct nd_options_header *nd_opts_ri_end; struct nd_options_header *user_opts; struct nd_options_header *user_opts_end; + std::array opt_array; bool is_useropt(struct nd_options_header *opt) { @@ -128,7 +128,7 @@ namespace icmp6 { uint8_t options[0]; uint16_t option_offset() - { return 16 * 2; } + { return IP6_ADDR_BYTES * 2; } } __attribute__((packed)); @@ -141,7 +141,7 @@ namespace icmp6 { { return target; } uint16_t option_offset() - { return 16; } + { return IP6_ADDR_BYTES; } } __attribute__((packed)); @@ -154,7 +154,7 @@ namespace icmp6 { { return target; } uint16_t option_offset() - { return 16; } + { return IP6_ADDR_BYTES; } } __attribute__((packed)); @@ -163,8 +163,7 @@ namespace icmp6 { public: - NdpPacket(Packet& icmp6) : icmp6_(icmp6), ndp_opt_() { - } + NdpPacket(Packet& icmp6) : icmp6_(icmp6), ndp_opt_() {} void parse(icmp6::Type type); @@ -217,7 +216,7 @@ namespace icmp6 { }; struct Header { - Type type; + Type type; uint8_t code; uint16_t checksum; union { diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index fd374352fa..4a0e537952 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -287,7 +287,6 @@ namespace net if (packet == nullptr) return; if (next_hop == IP6::ADDR_ANY) { - // Create local and target subnets addr target = packet->ip_dst() & stack_.netmask6(); addr local = stack_.ip6_addr() & stack_.netmask6(); diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index 45a1ff6f58..812d99bc8e 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -64,8 +64,6 @@ namespace net res.set_type(ICMP_type::ND_NEIGHBOUR_ADV); res.set_code(0); res.ndp().set_neighbour_adv_flag(NEIGH_ADV_SOL | NEIGH_ADV_OVERRIDE); - PRINT("NDP: Transmitting Neighbor adv to %s\n", - res.ip().ip_dst().str().c_str()); // Insert target link address, ICMP6 option header and our mac address res.set_payload({req.ndp().neighbour_sol().get_target().data(), 16 }); @@ -75,8 +73,9 @@ namespace net // Add checksum res.set_checksum(); - PRINT("NDP: Neighbor Adv Response size: %i payload size: %i, checksum: 0x%x\n", - res.ip().size(), res.payload().size(), res.compute_checksum()); + PRINT("NDP: Neighbor Adv Response dst: %s size: %i payload size: %i," + " checksum: 0x%x\n", res.ip().size(), res.payload().size(), + res.ip().ip_dst().str().c_str(), res.compute_checksum()); auto dest = res.ip().ip_dst(); transmit(res.release(), dest); @@ -146,6 +145,7 @@ namespace net req.ip().ip_dst().get_part(13), req.ip().ip_dst().get_part(14), req.ip().ip_dst().get_part(15)); + PRINT("NDP: Sending Neighbour solicit size: %i payload size: %i," "checksum: 0x%x\n, source: %s, dest: %s, dest mac: %s\n", req.ip().size(), req.payload().size(), req.compute_checksum(), @@ -243,7 +243,8 @@ namespace net { } - void Ndp::receive(icmp6::Packet& pckt) { + void Ndp::receive(icmp6::Packet& pckt) + { switch(pckt.type()) { case (ICMP_type::ND_ROUTER_SOL): PRINT("NDP: Router solictation message from %s\n", pckt.ip().ip_src().str().c_str()); @@ -331,7 +332,8 @@ namespace net } - void Ndp::await_resolution(Packet_ptr pckt, IP6::addr next_hop) { + void Ndp::await_resolution(Packet_ptr pckt, IP6::addr next_hop) + { auto queue = waiting_packets_.find(next_hop); PRINT(" Waiting for resolution of %s\n", next_hop.str().c_str()); if (queue != waiting_packets_.end()) { From 066eeb5152b919e1534bc4934cbc38c512c3f073 Mon Sep 17 00:00:00 2001 From: niks3089 Date: Tue, 29 May 2018 19:00:26 +0530 Subject: [PATCH 354/723] test: Add ndp.cpp to test cmake --- test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6e7099c504..a907136a56 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -205,6 +205,7 @@ set(OS_SOURCES ${SRC}/net/ip4/udp.cpp ${SRC}/net/ip4/udp_socket.cpp ${SRC}/net/ip6/icmp6.cpp + ${SRC}/net/ip6/ndp.cpp ${SRC}/net/ip6/ip6.cpp ${SRC}/net/dhcp/dh4client.cpp ${SRC}/net/nat/nat.cpp From d4c0a0d03610d2348968ba742f09e32c53c2fb80 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 30 May 2018 09:45:18 +0200 Subject: [PATCH 355/723] Test: Quiet apt-get for grub setup.sh --- test/kernel/integration/grub/setup.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/kernel/integration/grub/setup.sh b/test/kernel/integration/grub/setup.sh index 98481e0ebf..7b56559d4f 100755 --- a/test/kernel/integration/grub/setup.sh +++ b/test/kernel/integration/grub/setup.sh @@ -1,3 +1,2 @@ #!/bin/bash -sudo apt install -qy grub-pc xorriso - +sudo apt-get -q install -qy grub-pc xorriso From 62d8872f1dfcf6002307381640d12a475c623523 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 30 May 2018 09:45:36 +0200 Subject: [PATCH 356/723] Test: More readable stderr output in tests --- vmrunner/prettify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vmrunner/prettify.py b/vmrunner/prettify.py index b53bf5333a..a7a0a431d0 100644 --- a/vmrunner/prettify.py +++ b/vmrunner/prettify.py @@ -91,7 +91,7 @@ def VM(string): @staticmethod def DATA(string): - return color.C_GRAY + string + color.C_ENDC + "\n" + return string + "\n" @staticmethod def HEADER(string): From 4bdd0843fb357264e729797af5bbcc93c9586937 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 30 May 2018 10:46:27 +0200 Subject: [PATCH 357/723] test: Silence outout from paging unittest --- .../unit/{events_test.cpp => unit_events.cpp} | 0 test/kernel/unit/x86_paging.cpp | 5 +-- test/lest_util/common.cxx | 32 +++++++++---------- 3 files changed, 18 insertions(+), 19 deletions(-) rename test/kernel/unit/{events_test.cpp => unit_events.cpp} (100%) diff --git a/test/kernel/unit/events_test.cpp b/test/kernel/unit/unit_events.cpp similarity index 100% rename from test/kernel/unit/events_test.cpp rename to test/kernel/unit/unit_events.cpp diff --git a/test/kernel/unit/x86_paging.cpp b/test/kernel/unit/x86_paging.cpp index cd46f1d3d7..495bac55de 100644 --- a/test/kernel/unit/x86_paging.cpp +++ b/test/kernel/unit/x86_paging.cpp @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#define DEBUG_UNIT +//#define DEBUG_UNIT #include #include @@ -564,8 +564,9 @@ CASE ("x86::paging Verify default paging setup") using Tbl = std::array; Tbl& tbl4 = *(Tbl*)__pml4->data(); - +#ifdef DEBUG_UNIT std::cout << __pml4->summary(true, 0) << "\n"; +#endif // PML4 for (auto& ent4 : tbl4 ) { diff --git a/test/lest_util/common.cxx b/test/lest_util/common.cxx index 0b5f3a7836..2ef197b9ec 100644 --- a/test/lest_util/common.cxx +++ b/test/lest_util/common.cxx @@ -5,34 +5,32 @@ #define CASE( name ) lest_CASE( specification(), name ) extern lest::tests & specification(); -#define DEBUG_UNIT +//#define DEBUG_UNIT #ifdef DEBUG_UNIT #define LINK printf("%s:%i: OK\n",__FILE__,__LINE__) #else #define LINK (void) #endif -namespace test { - -uint64_t rand64(); -template -static std::vector rands64() +namespace test { - static std::vector rnd; - - if (rnd.empty()) + extern uint64_t rand64(); + template + static std::vector rands64() { - for (int i = 0; i < N; i++) + static std::vector rnd; + + if (rnd.empty()) { - rnd.push_back(rand64()); + for (int i = 0; i < N; i++) + { + rnd.push_back(rand64()); + } } - } - - return rnd; -} - -const std::vector random = rands64<10>(); + return rnd; + } + const std::vector random = rands64<10>(); } #ifndef HAVE_LEST_MAIN From b77fb6155ed8b2a7b68c2625eafec3e06c5384fe Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 30 May 2018 11:24:02 +0200 Subject: [PATCH 358/723] test: Add unittest for Timers system --- test/CMakeLists.txt | 5 +- .../integration/LiveUpdate/CMakeLists.txt | 6 +- test/kernel/unit/unit_timers.cpp | 87 +++++++++++++++++++ test/lest_util/os_mock.cpp | 18 ++-- 4 files changed, 101 insertions(+), 15 deletions(-) create mode 100644 test/kernel/unit/unit_timers.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6e7099c504..3c9395b2e9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -24,6 +24,7 @@ message(STATUS "Building for arch ${ARCH}") add_definitions(-DARCH_${ARCH}) add_definitions(-DARCH="${ARCH}") add_definitions(-DOS_VERSION="v0.0.0.1") +add_definitions(-DINCLUDEOS_SINGLE_THREADED) set(CMAKE_C_FLAGS "-g -O0 -std=c11 -Wall -Wextra") @@ -86,12 +87,13 @@ set(TEST_SOURCES #${TEST}/hw/unit/cpu_test.cpp ${TEST}/hw/unit/mac_addr_test.cpp ${TEST}/hw/unit/nic_test.cpp - ${TEST}/kernel/unit/events_test.cpp ${TEST}/kernel/unit/memmap_test.cpp ${TEST}/kernel/unit/memory.cpp ${TEST}/kernel/unit/x86_paging.cpp ${TEST}/kernel/unit/os_test.cpp ${TEST}/kernel/unit/service_stub_test.cpp + ${TEST}/kernel/unit/unit_events.cpp + ${TEST}/kernel/unit/unit_timers.cpp ${TEST}/net/unit/bufstore.cpp ${TEST}/net/unit/checksum.cpp ${TEST}/net/unit/cidr.cpp @@ -177,6 +179,7 @@ set(OS_SOURCES ${SRC}/kernel/rng.cpp ${SRC}/kernel/service_stub.cpp ${SRC}/kernel/syscalls.cpp + ${SRC}/kernel/timers.cpp ${SRC}/musl/mmap.cpp ${SRC}/musl/brk.cpp ${SRC}/net/buffer_store.cpp diff --git a/test/kernel/integration/LiveUpdate/CMakeLists.txt b/test/kernel/integration/LiveUpdate/CMakeLists.txt index 477ee3e0e7..82c965e700 100644 --- a/test/kernel/integration/LiveUpdate/CMakeLists.txt +++ b/test/kernel/integration/LiveUpdate/CMakeLists.txt @@ -24,11 +24,11 @@ set(SOURCES # DRIVERS / PLUGINS: set(DRIVERS virtionet - #boot_logger + boot_logger ) set(PLUGINS - system_log + #system_log ) # STATIC LIBRARIES: @@ -37,7 +37,7 @@ set(LIBRARIES ) # disable serial output -set(default_stdout OFF) +#set(default_stdout OFF) # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/kernel/unit/unit_timers.cpp b/test/kernel/unit/unit_timers.cpp new file mode 100644 index 0000000000..5a1d2e95fb --- /dev/null +++ b/test/kernel/unit/unit_timers.cpp @@ -0,0 +1,87 @@ +#include +#include +using namespace std::chrono; + +extern delegate systime_override; +static uint64_t current_time = 0; + +static int magic_performed = 0; +void perform_magic(int) { + magic_performed += 1; +} + +CASE("Initialize timer system") +{ + systime_override = + [] () -> uint64_t { return current_time; }; + + Timers::init( + [] (Timers::duration_t) {}, + [] () {} + ); + Timers::ready(); + + EXPECT(Timers::is_ready()); + EXPECT(Timers::active() == 0); + EXPECT(Timers::existing() == 0); + EXPECT(Timers::free() == 0); +} + +CASE("Start a single timer, execute it") +{ + current_time = 0; + magic_performed = 0; + // start timer + Timers::oneshot(1ms, perform_magic); + EXPECT(Timers::active() == 1); + EXPECT(Timers::existing() == 1); + EXPECT(Timers::free() == 0); + // execute timer interrupt + Timers::timers_handler(); + // verify timer did not execute + EXPECT(magic_performed == 0); + // set time to where it should happen + current_time = 1000000; + // execute timer interrupt + Timers::timers_handler(); + // verify timer did execute + EXPECT(magic_performed == 1); +} + +CASE("Start a single timer, execute it precisely") +{ + current_time = 0; + magic_performed = 0; + // start timer + Timers::oneshot(1ms, perform_magic); + // verify timer did not execute for all times before 1ms + for (uint64_t time = 0; time < 1000000; time += 1000) + { + Timers::timers_handler(); + EXPECT(magic_performed == 0); + } + // set time to where it should happen + current_time = 1000000; + // execute timer interrupt + Timers::timers_handler(); + // verify timer did execute + EXPECT(magic_performed == 1); +} + +CASE("Start many timers, execute all at once") +{ + current_time = 0; + magic_performed = 0; + // start many timers + for (int i = 0; i < 1000; i++) + Timers::oneshot(microseconds(i), perform_magic); + current_time = 1000 * 1000000ull; + // verify many timers executed + Timers::timers_handler(); + EXPECT(magic_performed == 1000); + EXPECT(Timers::active() == 0); + EXPECT(Timers::existing() == 1000); + EXPECT(Timers::free() == 1000); + // restore time + current_time = 0; +} diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index 56d0233f94..554c958b84 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -50,20 +50,15 @@ Statman& Statman::get() { RTC::timestamp_t RTC::booted_at = 0; void RTC::init() {} -/// TIMERS /// -#include -void Timers::timers_handler() {} -void Timers::ready() {} -void Timers::stop(int) {} -void Timers::init(const start_func_t&, const stop_func_t&) {} -Timers::id_t Timers::periodic(duration_t, duration_t, handler_t) { - return 0; -} - #include const char* service_binary_name__ = "Service binary name"; const char* service_name__ = "Service name"; +void Service::ready() +{ + printf("Service::ready() called\n"); +} + extern "C" void kprintf(char* format, ...) { @@ -186,8 +181,9 @@ void __arch_subscribe_irq(uint8_t) {} void __arch_enable_legacy_irq(uint8_t) {} void __arch_disable_legacy_irq(uint8_t) {} +delegate systime_override = [] () -> uint64_t { return 0; }; uint64_t __arch_system_time() noexcept { - return 0; + return systime_override(); } #include timespec __arch_wall_clock() noexcept { From 0799f191ab2534920709b48c744a31b1881f1fc9 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 30 May 2018 11:31:53 +0200 Subject: [PATCH 359/723] test: Add some extra checks to Timers unittest --- test/kernel/unit/unit_timers.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/kernel/unit/unit_timers.cpp b/test/kernel/unit/unit_timers.cpp index 5a1d2e95fb..87f31ba41f 100644 --- a/test/kernel/unit/unit_timers.cpp +++ b/test/kernel/unit/unit_timers.cpp @@ -72,10 +72,16 @@ CASE("Start many timers, execute all at once") { current_time = 0; magic_performed = 0; - // start many timers + // start many timers, starting at 1ms and ending at 2ms for (int i = 0; i < 1000; i++) - Timers::oneshot(microseconds(i), perform_magic); - current_time = 1000 * 1000000ull; + Timers::oneshot(microseconds(1000 + i), perform_magic); + // verify timer did not execute for all times before 1ms + for (uint64_t time = 0; time < 1000000; time += 1000) + { + Timers::timers_handler(); + EXPECT(magic_performed == 0); + } + current_time = 2000000; // verify many timers executed Timers::timers_handler(); EXPECT(magic_performed == 1000); From 30f14fc044d98e9bad0ac537553941bb20c8c7b0 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 30 May 2018 12:17:27 +0200 Subject: [PATCH 360/723] test: Add more Timers cases, license --- test/kernel/unit/unit_timers.cpp | 43 ++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/test/kernel/unit/unit_timers.cpp b/test/kernel/unit/unit_timers.cpp index 87f31ba41f..3edfe5a184 100644 --- a/test/kernel/unit/unit_timers.cpp +++ b/test/kernel/unit/unit_timers.cpp @@ -1,3 +1,20 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2016-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + #include #include using namespace std::chrono; @@ -91,3 +108,29 @@ CASE("Start many timers, execute all at once") // restore time current_time = 0; } + +CASE("Catch some Timers exceptions") +{ + EXPECT_THROWS(Timers::stop(-1)); +} + +CASE("Stop a timer") +{ + current_time = 0; + magic_performed = 0; + // start timer + int id = Timers::oneshot(1ms, perform_magic); + EXPECT(Timers::active() == 1); + // execute timer interrupt + Timers::timers_handler(); + // verify timer did not execute + EXPECT(magic_performed == 0); + // set time to where it should happen + current_time = 1000000; + // stop timer + Timers::stop(id); + // execute timer interrupt + Timers::timers_handler(); + // verify timer did not execute, since it was stopped + EXPECT(magic_performed == 0); +} From 891fff159dab8a72641b4534481c0bac2f951102 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 30 May 2018 12:24:31 +0200 Subject: [PATCH 361/723] test: Add periodic timer to Timers unittest --- test/kernel/unit/unit_timers.cpp | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/kernel/unit/unit_timers.cpp b/test/kernel/unit/unit_timers.cpp index 3edfe5a184..6f7df028a8 100644 --- a/test/kernel/unit/unit_timers.cpp +++ b/test/kernel/unit/unit_timers.cpp @@ -134,3 +134,24 @@ CASE("Stop a timer") // verify timer did not execute, since it was stopped EXPECT(magic_performed == 0); } + +CASE("Test a periodic timer") +{ + current_time = 0; + magic_performed = 0; + // start timer + int id = Timers::periodic(microseconds(1), perform_magic); + EXPECT(Timers::active() == 1); + + for (int i = 0; i < 100000; i += 1000) + { + current_time = i; + // execute timer interrupt + Timers::timers_handler(); + // verify timer did not execute + EXPECT(magic_performed == i / 1000); + } + // stop timer + Timers::stop(id); + current_time = 0; +} From d6c54a582332f2b2d471694b206a8a04fd12203d Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 30 May 2018 13:06:47 +0200 Subject: [PATCH 362/723] Stress test: Increase includeOS vm timeout + fire_bursts cleanup --- test/stress/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/stress/test.py b/test/stress/test.py index 6f09426a38..fa58f1889a 100755 --- a/test/stress/test.py +++ b/test/stress/test.py @@ -163,7 +163,7 @@ def crash_test(string): def fire_bursts(func, sub_test_name, lead_out = 3): name_tag = "<" + sub_test_name + ">" print color.HEADER(test_name + " initiating "+sub_test_name) - membase_start = func() + membase_start = get_mem() mem_base = membase_start # Track heap behavior @@ -262,7 +262,7 @@ def wait_for_tw(): vm.on_output("Ready to end", check_vitals) # Boot the VM, taking a timeout as parameter -timeout = BURST_COUNT * 20 +timeout = BURST_COUNT * 30 if len(sys.argv) > 1: timeout = int(sys.argv[1]) From 345db0c06a28b1fa4452033459f0dd53091e27fa Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 30 May 2018 13:08:36 +0200 Subject: [PATCH 363/723] Testrunner: Added unbuffered to all python commands, to improve output --- test/testrunner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/testrunner.py b/test/testrunner.py index 5583740fb1..5f308ea7c7 100755 --- a/test/testrunner.py +++ b/test/testrunner.py @@ -68,7 +68,7 @@ def print_skipped(tests): class Test: """ A class to start a test as a subprocess and pretty-print status """ - def __init__(self, path, clean=False, command=['python', 'test.py'], name=None): + def __init__(self, path, clean=False, command=['python', '-u', 'test.py'], name=None): self.command_ = command self.proc_ = None self.path_ = path From efbff3afbb6e8322948d50ad05a9dc2cabe8cb27 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 30 May 2018 16:00:13 +0200 Subject: [PATCH 364/723] LiveUpdate: Make unit-testable --- lib/LiveUpdate/liveupdate.hpp | 19 ++++++++++++++----- lib/LiveUpdate/partition.cpp | 3 ++- lib/LiveUpdate/resume.cpp | 12 ++++++------ lib/LiveUpdate/storage.cpp | 4 ++-- lib/LiveUpdate/storage.hpp | 8 ++++---- lib/LiveUpdate/update.cpp | 25 ++++++++++++++----------- 6 files changed, 42 insertions(+), 29 deletions(-) diff --git a/lib/LiveUpdate/liveupdate.hpp b/lib/LiveUpdate/liveupdate.hpp index 95993ef86a..99b7654590 100644 --- a/lib/LiveUpdate/liveupdate.hpp +++ b/lib/LiveUpdate/liveupdate.hpp @@ -58,7 +58,8 @@ struct LiveUpdate // Start a live update process, storing all user-defined data // If no storage functions are registered no state will be saved - static void exec(const buffer_t& blob); + // If @storage_area is nullptr (default) it will be retrieved from OS + static void exec(const buffer_t& blob, void* storage_area = nullptr); // Same as above, but including the partition [@key, func] static void exec(const buffer_t& blob, std::string key, storage_func func); @@ -74,7 +75,7 @@ struct LiveUpdate // It performs an extensive validation process to make sure the data is // complete and consistent static bool is_resumable(); - static bool is_resumable(void* location); + static bool is_resumable(const void* location); // Restore existing state for a partition named @key. // Returns false if there was no such partition @@ -82,7 +83,8 @@ struct LiveUpdate static bool resume(std::string key, resume_func handler); // Check whether a partition named @key exists at default update location. - static bool partition_exists(const std::string& key) noexcept; + // When @storage_area is nullptr (default) the area is retrieved from OS + static bool partition_exists(const std::string& key, const void* storage_area = nullptr) noexcept; // When explicitly resuming from heap, heap overrun checks are disabled static void resume_from_heap(void* location, std::string key, resume_func); @@ -90,8 +92,7 @@ struct LiveUpdate // Retrieve the recorded length, in bytes, of a valid storage area // Throws std::runtime_error when something bad happens // Never returns zero - static size_t stored_data_length(); - static size_t stored_data_length(void* location); + static size_t stored_data_length(const void* storage_area = nullptr); // Set location of known good blob to rollback to if something happens static void set_rollback_blob(const void*, size_t) noexcept; @@ -254,6 +255,14 @@ inline void Storage::add_vector(uid id, const std::vector& vector) add_string_vector(id, vector); } +class liveupdate_exec_success : public std::exception +{ +public: + const char* what() const throw() { + return "LiveUpdate::exec() success"; + } +}; + } // liu #endif diff --git a/lib/LiveUpdate/partition.cpp b/lib/LiveUpdate/partition.cpp index 0bb92f667d..2d5de6a7e2 100644 --- a/lib/LiveUpdate/partition.cpp +++ b/lib/LiveUpdate/partition.cpp @@ -21,6 +21,7 @@ #include "storage.hpp" #include #include +#include inline uint32_t liu_crc32(const void* buf, size_t len) { @@ -42,7 +43,7 @@ int storage_header::create_partition(const std::string key) part.offset = this->length; return partitions++; } -int storage_header::find_partition(const char* key) +int storage_header::find_partition(const char* key) const { for (uint32_t p = 0; p < this->partitions; p++) { diff --git a/lib/LiveUpdate/resume.cpp b/lib/LiveUpdate/resume.cpp index 0ea3730c1f..34a9bfd985 100644 --- a/lib/LiveUpdate/resume.cpp +++ b/lib/LiveUpdate/resume.cpp @@ -37,9 +37,9 @@ bool LiveUpdate::is_resumable() { return is_resumable(OS::liveupdate_storage_area()); } -bool LiveUpdate::is_resumable(void* location) +bool LiveUpdate::is_resumable(const void* location) { - return ((storage_header*) location)->validate(); + return ((const storage_header*) location)->validate(); } static bool resume_helper(void* location, std::string key, LiveUpdate::resume_func func) @@ -64,14 +64,14 @@ bool LiveUpdate::resume(std::string key, resume_func func) } return resume_helper(location, std::move(key), func); } -bool LiveUpdate::partition_exists(const std::string& key) noexcept +bool LiveUpdate::partition_exists(const std::string& key, const void* area) noexcept { - auto* location = OS::liveupdate_storage_area(); + if (area == nullptr) area = OS::liveupdate_storage_area(); - if (!LiveUpdate::is_resumable(location)) + if (!LiveUpdate::is_resumable(area)) return false; - auto& storage = *reinterpret_cast(location); + auto& storage = *(const storage_header*) area; return (storage.find_partition(key.c_str()) != -1); } void LiveUpdate::resume_from_heap(void* location, std::string key, LiveUpdate::resume_func func) diff --git a/lib/LiveUpdate/storage.cpp b/lib/LiveUpdate/storage.cpp index b54cf909a3..c510e34834 100644 --- a/lib/LiveUpdate/storage.cpp +++ b/lib/LiveUpdate/storage.cpp @@ -140,7 +140,7 @@ void storage_header::finalize() // generate checksum for header this->crc = generate_checksum(); } -bool storage_header::validate() noexcept +bool storage_header::validate() const noexcept { if (this->magic != LIVEUPD_MAGIC) return false; if (this->crc == 0) return false; @@ -150,7 +150,7 @@ bool storage_header::validate() noexcept return true; } -uint32_t storage_header::generate_checksum() noexcept +uint32_t storage_header::generate_checksum() const noexcept { uint32_t crc_copy = this->crc; this->crc = 0; diff --git a/lib/LiveUpdate/storage.hpp b/lib/LiveUpdate/storage.hpp index 562578df27..de822864bd 100644 --- a/lib/LiveUpdate/storage.hpp +++ b/lib/LiveUpdate/storage.hpp @@ -116,7 +116,7 @@ struct storage_header storage_header(); int create_partition(std::string key); - int find_partition(const char*); + int find_partition(const char*) const; void finish_partition(int); void zero_partition(int); @@ -143,17 +143,17 @@ struct storage_header ((storage_entry*) &vla[length])->type = TYPE_END; } void finalize(); - bool validate() noexcept; + bool validate() const noexcept; // zero out everything if all partitions consumed void try_zero() noexcept; private: - uint32_t generate_checksum() noexcept; + uint32_t generate_checksum() const noexcept; // zero out the entire header and its data, for extra security void zero(); uint64_t magic; - uint32_t crc; + mutable uint32_t crc; uint32_t entries = 0; uint32_t length = 0; std::array ptable; diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index 71dbd3713e..3de76b1fa4 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -87,12 +87,16 @@ void LiveUpdate::exec(const buffer_t& blob, std::string key, storage_func func) LiveUpdate::exec(blob); } -void LiveUpdate::exec(const buffer_t& blob) +void LiveUpdate::exec(const buffer_t& blob, void* location) { - void* location = OS::liveupdate_storage_area(); + if (location == nullptr) location = OS::liveupdate_storage_area(); LPRINT("LiveUpdate::begin(%p, %p:%d, ...)\n", location, blob.data(), (int) blob.size()); +#if defined(PLATFORM_x86_solo5) || defined(PLATFORM_UNITTEST) + // nothing to do +#else // 1. turn off interrupts asm volatile("cli"); +#endif // use area provided to us directly, which we will assume // is far enough into heap to not get overwritten by hotswap. @@ -101,6 +105,8 @@ void LiveUpdate::exec(const buffer_t& blob) // blobs are separated by at least one old kernel size and // some early heap allocations, which is at least 1mb, while // the copy mechanism just copies single bytes. + if (blob.size() < ELF_MINIMUM) + throw std::runtime_error("Buffer too small to be valid ELF"); const char* update_area = blob.data(); char* storage_area = (char*) location; const uintptr_t storage_area_phys = os::mem::virt_to_phys((uintptr_t) storage_area); @@ -204,7 +210,7 @@ void LiveUpdate::exec(const buffer_t& blob) hw::Devices::deactivate_all(); // store soft-resetting stuff -#ifdef PLATFORM_x86_solo5 +#if defined(PLATFORM_x86_solo5) || defined(PLATFORM_UNITTEST) void* sr_data = nullptr; #else extern const std::pair get_rollback_location(); @@ -229,8 +235,9 @@ void LiveUpdate::exec(const buffer_t& blob) #ifdef PLATFORM_x86_solo5 solo5_exec(blob.data(), blob.size()); throw std::runtime_error("solo5_exec returned"); -#else -# ifdef ARCH_i686 +# elif defined(PLATFORM_UNITTEST) + throw liveupdate_exec_success(); +# elif defined(ARCH_i686) // copy hotswapping function to sweet spot memcpy(HOTSWAP_AREA, (void*) &hotswap, &__hotswap_length - (char*) &hotswap); /// the end @@ -245,7 +252,6 @@ void LiveUpdate::exec(const buffer_t& blob) # else # error "Unimplemented architecture" # endif -#endif } void LiveUpdate::restore_environment() { @@ -259,12 +265,9 @@ buffer_t LiveUpdate::store() return buffer_t(location, location + size); } -size_t LiveUpdate::stored_data_length() -{ - return stored_data_length(OS::liveupdate_storage_area()); -} -size_t LiveUpdate::stored_data_length(void* location) +size_t LiveUpdate::stored_data_length(const void* location) { + if (location == nullptr) location = OS::liveupdate_storage_area(); auto* storage = (storage_header*) location; if (LIVEUPDATE_PERFORM_SANITY_CHECKS) From 28ef373bb5940a45f6a0dbb40542fd6236e5d8d3 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 30 May 2018 16:00:35 +0200 Subject: [PATCH 365/723] test: Add simple LiveUpdate unittest --- test/CMakeLists.txt | 15 +++++++ test/kernel/unit/unit_liveupdate.cpp | 61 ++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 test/kernel/unit/unit_liveupdate.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3c9395b2e9..d429488b0c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -23,6 +23,7 @@ message(STATUS "Building for arch ${ARCH}") add_definitions(-DARCH_${ARCH}) add_definitions(-DARCH="${ARCH}") +add_definitions(-DPLATFORM_UNITTEST) add_definitions(-DOS_VERSION="v0.0.0.1") add_definitions(-DINCLUDEOS_SINGLE_THREADED) @@ -69,6 +70,7 @@ include_directories( ${INCLUDEOS_ROOT}/mod/ ${INCLUDEOS_ROOT}/mod/GSL ${INCLUDEOS_ROOT}/mod/uzlib/src + ${INCLUDEOS_ROOT}/lib/LiveUpdate ${TEST}/lest/include $ENV{INCLUDEOS_PREFIX}/include ) @@ -94,6 +96,7 @@ set(TEST_SOURCES ${TEST}/kernel/unit/service_stub_test.cpp ${TEST}/kernel/unit/unit_events.cpp ${TEST}/kernel/unit/unit_timers.cpp + ${TEST}/kernel/unit/unit_liveupdate.cpp ${TEST}/net/unit/bufstore.cpp ${TEST}/net/unit/checksum.cpp ${TEST}/net/unit/cidr.cpp @@ -238,11 +241,23 @@ set(OS_SOURCES ) set(MOD_OBJECTS + # http-parser ${INCLUDEOS_ROOT}/mod/http-parser/http_parser.c + # uzlib ${INCLUDEOS_ROOT}/mod/uzlib/src/adler32.c ${INCLUDEOS_ROOT}/mod/uzlib/src/crc32.c ${INCLUDEOS_ROOT}/mod/uzlib/src/tinflate.c ${INCLUDEOS_ROOT}/mod/uzlib/src/tinfgzip.c + # LiveUpdate + #${INCLUDEOS_ROOT}/lib/LiveUpdate/hotswap.cpp + ${INCLUDEOS_ROOT}/lib/LiveUpdate/partition.cpp + ${INCLUDEOS_ROOT}/lib/LiveUpdate/rollback.cpp + ${INCLUDEOS_ROOT}/lib/LiveUpdate/serialize_tcp.cpp + ${INCLUDEOS_ROOT}/lib/LiveUpdate/update.cpp + ${INCLUDEOS_ROOT}/lib/LiveUpdate/os.cpp + ${INCLUDEOS_ROOT}/lib/LiveUpdate/resume.cpp + #${INCLUDEOS_ROOT}/lib/LiveUpdate/serialize_openssl.cpp + ${INCLUDEOS_ROOT}/lib/LiveUpdate/storage.cpp ) if(EXTRA_TESTS) diff --git a/test/kernel/unit/unit_liveupdate.cpp b/test/kernel/unit/unit_liveupdate.cpp new file mode 100644 index 0000000000..ad01f0fae8 --- /dev/null +++ b/test/kernel/unit/unit_liveupdate.cpp @@ -0,0 +1,61 @@ +#include +#include +#include +using namespace liu; + +static buffer_t not_a_kernel; +static void* storage_area = nullptr; +static struct { + int integer = 0; + std::string string = ""; + bool boolean = false; +} stored; + +static void store_something(Storage& store, const buffer_t*) +{ + store.add_int(0, 1234); + store.add_string(1, "Test string"); + store.add (2, true); +} +static void restore_something(Restore& thing) +{ + stored.integer = thing.as_int(); thing.go_next(); + stored.string = thing.as_string(); thing.go_next(); + stored.boolean = thing.as_type(); thing.go_next(); +} + +CASE("Setup LiveUpdate and perform no-op update") +{ + storage_area = new char[16*1024*1024]; + + not_a_kernel.resize(164); + auto* elf = (Elf64_Ehdr*) not_a_kernel.data(); + elf->e_ident[0] = 0x7F; + elf->e_ident[1] = 'E'; + elf->e_ident[2] = 'L'; + elf->e_ident[3] = 'F'; + elf->e_entry = 0x7F; + elf->e_phoff = 64; + elf->e_shnum = 0; + elf->e_shoff = 164; + + auto* phdr = (Elf64_Phdr*) ¬_a_kernel[elf->e_phoff]; + phdr->p_filesz = 164; + phdr->p_paddr = 0x80; + + EXPECT_THROWS_AS(LiveUpdate::exec(not_a_kernel, storage_area), liveupdate_exec_success); +} + +CASE("Store some data and restore it") +{ + LiveUpdate::register_partition("test", store_something); + EXPECT_THROWS_AS(LiveUpdate::exec(not_a_kernel, storage_area), liveupdate_exec_success); + + EXPECT(LiveUpdate::partition_exists("test", storage_area)); + + LiveUpdate::resume_from_heap(storage_area, "test", restore_something); + + EXPECT(stored.integer == 1234); + EXPECT(stored.string == "Test string"); + EXPECT(stored.boolean == true); +} From 373e73e6600bb50c4c3fb11cccf40cbf9c622c17 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 31 May 2018 12:32:31 +0200 Subject: [PATCH 366/723] linux: Update to latest dev, add debugging options --- cmake/linux.service.cmake | 10 ++++++++-- linux/CMakeLists.txt | 5 +++++ linux/src/main.cpp | 2 +- linux/userspace/CMakeLists.txt | 10 ++++++++-- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index c28d934cc4..ab371ee130 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -3,16 +3,22 @@ #################################### set(CMAKE_CXX_STANDARD 17) -set(COMMON "-g -O2 -march=native -Wall -Wextra") +set(COMMON "-O2 -march=native -Wall -Wextra") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") +option(DEBUGGING "Enable debugging" OFF) option(GPROF "Enable profiling with gprof" OFF) option(SANITIZE "Enable undefined- and address sanitizers" OFF) option(ENABLE_LTO "Enable thinLTO for use with LLD" OFF) option(CUSTOM_BOTAN "Enable building with a local Botan" OFF) option(STATIC_BUILD "Build a portable static executable" OFF) option(STRIP_BINARY "Strip final binary to reduce size" ON) +option(USE_LLD "Allow linking against LTO archives" ON) + +if(DEBUGGING) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") +endif() if (ENABLE_LTO) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") @@ -78,7 +84,7 @@ if (STATIC_BUILD) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -pthread") endif() -if (ENABLE_LTO) +if (ENABLE_LTO OR USE_LLD) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fuse-ld=lld") endif() diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index 37da393faa..2b11d0efbe 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -12,11 +12,16 @@ set(COMMON "-g -O2 -march=native -Wall -Wextra") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") +option(DEBUGGING "Enable debugging" OFF) option(GPROF "Enable profiling with gprof" OFF) option(SANITIZE "Enable undefined- and address sanitizers" OFF) option(ENABLE_LTO "Enable thinLTO for use with LLD" OFF) option(CUSTOM_BOTAN "Enable building with a local Botan" OFF) +if(DEBUGGING) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") +endif() + if (ENABLE_LTO) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") diff --git a/linux/src/main.cpp b/linux/src/main.cpp index 54c7c196fa..8f0f34aac2 100644 --- a/linux/src/main.cpp +++ b/linux/src/main.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include "drivers/tap_driver.hpp" #include "drivers/usernet.hpp" diff --git a/linux/userspace/CMakeLists.txt b/linux/userspace/CMakeLists.txt index 91e25408b4..6750a700e3 100644 --- a/linux/userspace/CMakeLists.txt +++ b/linux/userspace/CMakeLists.txt @@ -4,18 +4,22 @@ set(NET_SOURCES ${IOS}/src/net/buffer_store.cpp ${IOS}/src/net/checksum.cpp ${IOS}/src/net/super_stack.cpp - ${IOS}/src/net/inet4.cpp + ${IOS}/src/net/inet.cpp ${IOS}/src/net/ethernet/ethernet.cpp ${IOS}/src/net/ip4/arp.cpp ${IOS}/src/net/ip4/ip4.cpp ${IOS}/src/net/ip4/reassembly.cpp + ${IOS}/src/net/ip6/ip6.cpp + ${IOS}/src/net/ip6/icmp6.cpp + #${IOS}/src/net/ip6/ndp.cpp ${IOS}/src/net/tcp/tcp.cpp ${IOS}/src/net/tcp/connection.cpp ${IOS}/src/net/tcp/connection_states.cpp ${IOS}/src/net/tcp/write_queue.cpp ${IOS}/src/net/tcp/read_buffer.cpp + ${IOS}/src/net/tcp/read_request.cpp ${IOS}/src/net/tcp/rttm.cpp ${IOS}/src/net/tcp/listener.cpp ${IOS}/src/net/ip4/icmp4.cpp @@ -94,7 +98,9 @@ add_library(includeos STATIC ${NET_SOURCES} ${OS_SOURCES} ${MOD_SOURCES}) set_target_properties(includeos PROPERTIES LINKER_LANGUAGE CXX) target_include_directories(includeos PUBLIC ${IOS}/lib/mana/include - ${IOS}/mod/rapidjson/include) + ${IOS}/mod/rapidjson/include + $ENV{INCLUDEOS_PREFIX}/includeos/x86_64/include + ) add_library(http_parser STATIC "${IOS}/mod/http-parser/http_parser.c") set_target_properties(http_parser PROPERTIES LINKER_LANGUAGE C) From 795ea43820089521932b8f5320559be1deda0174 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 31 May 2018 12:34:08 +0200 Subject: [PATCH 367/723] net: Silent return on failed dynamic cast to ICMP error --- src/net/inet.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/net/inet.cpp b/src/net/inet.cpp index 05d7aeab3c..16b9da16ce 100644 --- a/src/net/inet.cpp +++ b/src/net/inet.cpp @@ -141,6 +141,9 @@ void Inet::error_report(Error& err, Packet_ptr orig_pckt) { if (err.is_icmp()) { auto* icmp_err = dynamic_cast(&err); + if (icmp_err == nullptr) { + return; // not an ICMP error + } if (icmp_err->is_too_big()) { // If Path MTU Discovery is not enabled, ignore the ICMP Datagram Too Big message From cae094da27f91d50a1d97159197c9e1b04720dff Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 31 May 2018 13:41:18 +0200 Subject: [PATCH 368/723] linux: Add websockets benchmark, OpenSSL, Botan2 support --- cmake/linux.service.cmake | 6 +- linux/src/os.cpp | 2 +- linux/userspace/CMakeLists.txt | 11 ++ test/linux/websockets/.gitignore | 3 + test/linux/websockets/CMakeLists.txt | 24 ++++ test/linux/websockets/callgraph.sh | 12 ++ test/linux/websockets/debug.sh | 5 + test/linux/websockets/memdisk.fat | Bin 0 -> 13312 bytes test/linux/websockets/run.sh | 37 ++++++ test/linux/websockets/service.cpp | 166 +++++++++++++++++++++++++++ 10 files changed, 262 insertions(+), 4 deletions(-) create mode 100644 test/linux/websockets/.gitignore create mode 100644 test/linux/websockets/CMakeLists.txt create mode 100755 test/linux/websockets/callgraph.sh create mode 100755 test/linux/websockets/debug.sh create mode 100644 test/linux/websockets/memdisk.fat create mode 100755 test/linux/websockets/run.sh create mode 100644 test/linux/websockets/service.cpp diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index ab371ee130..c20eb8299f 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -13,7 +13,7 @@ option(SANITIZE "Enable undefined- and address sanitizers" OFF) option(ENABLE_LTO "Enable thinLTO for use with LLD" OFF) option(CUSTOM_BOTAN "Enable building with a local Botan" OFF) option(STATIC_BUILD "Build a portable static executable" OFF) -option(STRIP_BINARY "Strip final binary to reduce size" ON) +option(STRIP_BINARY "Strip final binary to reduce size" OFF) option(USE_LLD "Allow linking against LTO archives" ON) if(DEBUGGING) @@ -34,7 +34,7 @@ if(SANITIZE) endif() if(CUSTOM_BOTAN) - include_directories("/usr/local/botan/include/botan-2") + include_directories("/usr/local/include/botan/botan-2") endif() add_definitions(-DARCH="x86_64" -DARCH_x86_64) @@ -74,7 +74,7 @@ add_executable(service ${SOURCES} ${IOSPATH}/src/service_name.cpp) set_target_properties(service PROPERTIES OUTPUT_NAME ${BINARY}) if (CUSTOM_BOTAN) - target_link_libraries(service /usr/local/botan/lib/libbotan-2.a -ldl -pthread) + target_link_libraries(service /usr/local/lib/libbotan-2.a -ldl -pthread) endif() target_link_libraries(service ${EXTRA_LIBS}) target_link_libraries(service includeos linuxrt includeos http_parser rt) diff --git a/linux/src/os.cpp b/linux/src/os.cpp index 0f798a340c..37a8e6a3df 100644 --- a/linux/src/os.cpp +++ b/linux/src/os.cpp @@ -153,5 +153,5 @@ extern "C" void panic(const char* why) { printf("!! PANIC !!\nReason: %s\n", why); - std::abort(); + raise(SIGINT); } diff --git a/linux/userspace/CMakeLists.txt b/linux/userspace/CMakeLists.txt index 6750a700e3..f78d4665fc 100644 --- a/linux/userspace/CMakeLists.txt +++ b/linux/userspace/CMakeLists.txt @@ -22,6 +22,7 @@ set(NET_SOURCES ${IOS}/src/net/tcp/read_request.cpp ${IOS}/src/net/tcp/rttm.cpp ${IOS}/src/net/tcp/listener.cpp + ${IOS}/src/net/tcp/stream.cpp ${IOS}/src/net/ip4/icmp4.cpp ${IOS}/src/net/ip4/udp.cpp ${IOS}/src/net/ip4/udp_socket.cpp @@ -35,6 +36,7 @@ set(NET_SOURCES ${IOS}/src/net/nat/nat.cpp ${IOS}/src/net/nat/napt.cpp + ${IOS}/src/net/http/basic_client.cpp ${IOS}/src/net/http/header.cpp ${IOS}/src/net/http/header_fields.cpp ${IOS}/src/net/http/message.cpp @@ -50,7 +52,16 @@ set(NET_SOURCES ${IOS}/src/net/http/server_connection.cpp ${IOS}/src/net/http/server.cpp ${IOS}/src/net/http/response_writer.cpp + + ${IOS}/src/net/https/botan_server.cpp + ${IOS}/src/net/https/openssl_server.cpp + ${IOS}/src/net/ws/websocket.cpp + + ${IOS}/src/net/openssl/init.cpp + ${IOS}/src/net/openssl/client.cpp + ${IOS}/src/net/openssl/server.cpp + ) if (CUSTOM_BOTAN) set(NET_SOURCES ${NET_SOURCES} diff --git a/test/linux/websockets/.gitignore b/test/linux/websockets/.gitignore new file mode 100644 index 0000000000..4ca28b0573 --- /dev/null +++ b/test/linux/websockets/.gitignore @@ -0,0 +1,3 @@ + +gmon.out +tcp_callgraph.png diff --git a/test/linux/websockets/CMakeLists.txt b/test/linux/websockets/CMakeLists.txt new file mode 100644 index 0000000000..75cacd86f4 --- /dev/null +++ b/test/linux/websockets/CMakeLists.txt @@ -0,0 +1,24 @@ +cmake_minimum_required(VERSION 2.8.9) +if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) + set(ENV{INCLUDEOS_PREFIX} /usr/local) +endif() +project (service C CXX) + +# Human-readable name of your service +set(SERVICE_NAME "Linux userspace WebSocket test") + +# Name of your service binary +set(BINARY "websockets") + +# Source files to be linked with OS library parts to form bootable image +set(SOURCES + service.cpp + ) + +set(EXTRA_LIBS + $ENV{INCLUDEOS_PREFIX}/includeos/x86_64/lib/libcrypto.a + $ENV{INCLUDEOS_PREFIX}/includeos/x86_64/lib/libssl.a + -ldl + ) + +include($ENV{INCLUDEOS_PREFIX}/includeos/linux.service.cmake) diff --git a/test/linux/websockets/callgraph.sh b/test/linux/websockets/callgraph.sh new file mode 100755 index 0000000000..4ed8e1114a --- /dev/null +++ b/test/linux/websockets/callgraph.sh @@ -0,0 +1,12 @@ +#! /bin/bash + + +echo -e "\n>>> Installing dependencies" +pip -q install gprof2dot +sudo apt install gprof graphviz + +echo -e "\n>>> Running tcp test service with profiling" +./run.sh gprof + +echo -e "\n>>> Generating graph png" +gprof ./build/linux_tcp | gprof2dot | dot -Tpng -o tcp_callgraph.png diff --git a/test/linux/websockets/debug.sh b/test/linux/websockets/debug.sh new file mode 100755 index 0000000000..4c46625cb7 --- /dev/null +++ b/test/linux/websockets/debug.sh @@ -0,0 +1,5 @@ +#!/bin/bash +pushd build +make -j8 +popd +sudo gdb build/websockets diff --git a/test/linux/websockets/memdisk.fat b/test/linux/websockets/memdisk.fat new file mode 100644 index 0000000000000000000000000000000000000000..03100b662f38e3272b0feb5b38fa6828f930a201 GIT binary patch literal 13312 zcmeHtdHD2JUGMvLgQgTJEsH|w3uTY(WHL!+0!Ya=NhXtZGMUM+W-{3(`!dN)1fdJa zyYMN&-Z)m*fCzBrgK`J9Xsv#@nfeQ>%w=BpZq@+@KgVfI(E-4Ut&kQjnid3HYW4| zd`faEfA`eMPljim`v0N7o}b1}&r6dw?Ri1v@7BG>S(m9DI#t=*6hZBr%Kg78|3g*% z*vF%P6VB*zo&22)C-FpO4F3F?KfqIhlYjiye%GDxo0^-yder5+<_>(v9r(cL z38%7r;*#6(ygLirQT`t;f7+=O;qu$d-xc5%I|aR%DZ(9oUH$L`LTc9zq?-a|LccKP6m=;dFh0r z@dlwYCj`bkl?sSFKT%blBxph+y)gH~fW;e#BvfWh5HpIJm5t|x3E?pWk@6IwQoBGz zxJ6l3i;9T>%#=N9&sA0Atdmi*Sr8zU)S0pv4seP0<51kn6G5>_yJif1N=}I-eW#SQ!o?O26WtG*Ri+Q>u{ki0&L$K_V@U(vFp(~hlg!5kI5-A(v@H~jwI3Y zY^PVGcmuQ$i|r@epuZ(Y{6LdzyG{v^?xEB`H{!VzjRnJO&OQ^q`$zF}N&c48l;Ayn?pO@(^ZlsTdVIo#+uBLWVYE78{21!X2!M zoS7~F5#LJ~5?KVT599@#%|bC_Qc}ghLCj#nsOzR1lCbFJLvX|Q5H1UHmo`F<*+OF5 zYyfS!-dU)nfQUI0ri?W~A!y8E^`VWmAe{;nsaQqX6A(VbcUzvE?J%NPK)Ll(-GH~{6KX2E3d5~3WEiU3@yPVyd_4g& zaLtT-PZYP<9%RwwtkrG4PU32zNm)ure%8*-D0PFC;#x{%Zg<2`m@S%U##jXsiGV4t zYPYn*McXHLh8g>Ukr}1BRHY?%$nuTFNNEn6bK4m7r+hz?q1wZmVPj=xD3`-E5VK-j zj$zzWkRq9NU&5Bk)fN^cbV2et$6Hko+D^Rrd_N9zeL(Ag0Z;Z; zBlHlMS_EYdQ8C@7^F@DEN;co{G@%mYsVPj;G}~)gYM%rU=I@<(p==eM2B^9~TgKjb zb6r_(5VyNi+D2aSACtDd?Oi4<}`sGJRRfXB2LW;kfM z&@vMr4VSZw#M&{u9qgh2ZvrE32WD0;%ZePBZZ$xvc8)}ok)#(rnq`(i!+EkX;%Q@_ z<5n@&1Bq9g!!V|?nJ4Z`0XsyAoA*4*$`Cuv>taofoyZ>zlaln)UIM_NrNM-g9t1&4 zM3P4nRd0lO0?sOJME4DB?-33Jvg}fi=S@Kf)TTw|m|PYFwx%{UK=k`8E_s{fg5N3o zja}{!K{K*?t5}TVq?n7If_jQmtc0FK5DQo~H^0xK?FW@!WKBNCnk={Z8?H8id_xJ8^spAcGneh^=M_U^bR@GHHZv z=UF8r)2VK|roHKWq}M?rrvPCox|b9Zyh7KR5VAWf+QB>$ zX>pP(h*1mU?N}-tUEVi5$SMPNttUmy*q*f*r&wkHrCOKn((Dx*3Tr~n4Wk2;J923? zD=EDE-$y4kcR#q}7PVbN%` zo(B72YUL|MUiCdw#r!xK8weYj9%IhUhFR43ECb{+rUxl^e2t=$ z?M5GlW71q;-7?!0HMMKa*hT{BAnl9IJf8BSEuhVEP3Gq4WHm?RLNAbEq%5~{5KAc- z+tqrq^}UfUtW<8$<83LrV}QwOFl9i6a{;Ns#_(E3hu92{Wo@6-B+q#)vKwq9dAC*B znvsaQAF(rwtT?l0sWoci3W!TkZ<=sAuo({~OGV#RI9~4PRVoOo9xBn4fFjz91%$9? z`{^7n62aT|;$n*ga5y*&ryC8Uw;-+n__D|9Jz=v*7Z7+@k4NcZ*urJQBH`L!`c4$4 zt-cE^UsOSE56<#I(~se}kE|8Yhv5_mbKCZblxq7DC!SZ zp=~2Gb2{Zivl9C2Rx~Jx!4}H{sR7<0*`=}~EEa|{vxKFMtT=OHZ8eZJq{V8Mu=J)$ zyt?tKzM_S}j3DOzGUID&E-3h}07i>>FbR=0rnSkb5hkW^BCJNGqp7&$io()CCh$~} z;34ISn%2{0Ft^yz2y+7hT1LPghqP@v@e^i|>sg2pENPD?^ptY8yIKt;so#QpFOO?* zMRD^;aI{{t9uG7p0dYf)O&}m$P}rfMC0pd2Sujw7O)6<=CgWkV(o}vL9yZa~J1mNk zzZ=WFdH}`~=#cNHV_iwxIS@0V+35R4&lpbBHqB#;>GdOJ)*Zl6(jAT~J;rnzMI>v& z+30Ax&dgX5%K=19hf$-hfz_szSws{Ql$*CmRFFe?A}`ZOc~V6~Q0td?WP(EXR*|)R&3Re;O^u` z{NBo6o6h1OSegnP>PUlYyv=EFBWaR0srLe$(s}4ka%r-6iRmEsvMsXfc80#nL&Sz@ z zRV=RtM~phK0bCWj`!9*95XYjWy~Fm?oH@|pN}BEzLS2WFyHtrm_Yp?$CM)w2#j}CE zLg-9|nW4Y}@&Uz$vv@L~$uunMep?Uan8dpC(%3CGwp$t25dEoFp z=Zl7kry>jN!oBBBLfA>mv4+T&Gc)xfsuu02+ZpN1!h=+pO0BS3#JlY_D+$O_kU>fn zo;m8z<{p65ZHP324{Dr6f`XaU*Wi)CVhTGr~mDC!P)3WMqX;q5z9xV*q zy3ni)SgS1O_I8prV5&{s+xdoFo3l;I&F2nV5ao(IY#&s>8>1_`dfa9aSBk@uyw>4&U}? zM@R4L@S7hzI{N1hum14S5z?`dU;60LQPklXA3HjFR)-Jm@ahh$k9Ym)Q2*S~(LZ$f zrOzK7ozW@3^vzxU9d7=n>raPg{O!@vb344c!#8wTefj9H2k6_nx?of4)x7xAe`}2-J;2w9o_`$t(ko4A8=>uuzBxM<$ zD4hJnC5h9VRJ)7PdJF>LSeKA)obtparEj5q_{1f96v9tjvat^3i2;T}cgfRTb>c3i z{F1JUyArzRyPCMYbSHAtB0h0a|Gy+ieyH4Cg*wq)h5ontbw^Krz*hi2nKs?Dfv!07 zY0wp?bFVo4wBx6pcJ{G%zWJu{t@4LJ=-ji;e)My==X~ef-6~< zXPtiS!GpD|H05jCli5t+@DV>r%0UY#3I;qtbchzsSYQm z)dqUd{m$)=AOs%v`+XdRR`)v>J{d*si2i3abH!;7`QaBhe&(^$uQ=^q$Dk`t15Uf* zwByJA{41|K{+z!mm#s}VZ;W?K-;XRjzHwhH~t~1~Cq+h%2;@RiOkCCrC{iW~zmtT9^y?_4dUnkth zAy1R9dd>Bh4xan)PyUbOS+AdeTK}8Bd=GMUVlQ9Iedxa!FaOk&Z@tey-;gQOH?%MQ z_FtmUeq?gbi#~PpKR)20>Steg;|`@j9@VJ8a@cl0vmp=937ry_K*FEo#7WTD|_l?uP_8{ZUFTZ8?yJvs%buanM zryhQ{Pb05-)%{-Red&VNe?Po#{q$>oQTf)+Y5eQ|CKjh^~8t0>wD-^UVQd9o_Wy|vFASf5sy88@EgzXfBjL<{_5rJ%ZJ()&-?T#P2_aiSZAA8L!UVT`6{uA&0#z!72 zd=Gr~wd-^5dG~L>w%?3C@%Mjs-G{yeU|;>hAHVk7=e_Cc_r2k#ey6m){KTsq`;%9G z^&^kZzVg`jJ>}IOdDxW?{)~Txs6;m%d;Gco?d!(M&at8P1SVtDVjzwwpZkDinM)!qN;QR2@kx6HNp+F!WW-`@~C?-K2^XY$v* z?_<{6-*O)O$v;-y7utS z*PVIm#m~W&v#xx?{zg+vX*b&E{6qTO*ZkB8_}QO&Efh`N z;v3;7F8jrgaxbH=KI;CvJS)HLtql^{@QQ_DiqF{@}lEuf4JS;>{m?>KE`~cEPW{{ha(B&)3Dv-t*jx zzSiBxzWJ0Zf8qFVdXM?~xqtGM+dlo*zxRRfKYxfn?aT+g7<&BIuXqFTk)M6zZC{qJ zyzIdmo&A@^0VZK6=TkKL6H7&CWW1^o)Iy`wxA@(_eAX>Yk4rUH|dp z?|#PM`|tYb`A;M6_p0CgR(litclZ9pC9nTm<2fI{fVt+D^t`9ueBOgT^$X2KmwoM7 z+vtsd^pKx<`g`IRefZZt{*JF0f2f{uR`0uy^G|=ot;LA1=G3j~-{XIqe~6P( zf2u+;2#{;i4!5IDF(;}1T*wHtCVMLZVv^lFX2!8XYo3nG!d-y00kvLbFh=Cja&5?K zv)Rt(z&e)9Y`#?k!*bwgP>$V}u*alWaQPCiyuLs4ceLNLP?uj%#+)13^W84T_oTG! zTL)zi6qBLLHheusp~Pp_!$^+Ya41nmn37@JE63!1Q4vc5lu8mT3IPwYor{K(s3In) z-HU-rYbTXv8a7+1@^maE8;(<_jlKkT#UPpu4iamP2}3d)7q3i?nAZyuAIAf4=~8@! zP{6WN3Ceo0S?-x5SnblCSaFhzYn6$UMTXzI?GZW+Cky4ns0NMSsWH1xHTLg zYLeF7<7+7m#*?-iYu#%-DGrLLx6_UewrI}9#9NVeXXhm`>RtLQG zaI*!KOoxPB0LJ>kr$}d1O!21TtRLooi5=thI1Q2~u_^TGB1I@Sk$E7UjiQF#Q!VKi zeTAf^ZW{KcOs`yTI)%xN;8aY#c`=^vF{h@{CFy`RA`qI0QP)5Xs>7(GxF-w{2r)ymbjKwxbdn4}Nc6FG zXiw>Uh8)C0K=36d8h-bQdi$c!55#nob<|;|w=M{`jS8$hs)wtrD0z+pTFX2i5KDK& zSzDpDlrh5Pz1@Ta5!bF8%jz#%v!8n8Y_{Eqelyz81OV%fKHGBcASXQ{Kp{dvyJaT? z95q#-VXxAA)mE!0nj#i5menS8nB_Aovxz7k)LUcBwTJ*ZH=Z#aslV8oj)j|t(N4omcjL_Q za@^~0<)$3buuQrP$Bw#9LEUzw!iHMHTPVa71WW*8=%N%X*`-d;T;4BbZXwr)l^q6R zHAmM&3TkK2!MZMNP7TGQzXoFUb7 zcuG(xIPZLoR1pb^oy*GZuyx9bAt2%2NDr2?1PX>wvMDHy-ONCcha7Lz2^(Rqu`g-cD0p-3aWxVfWN8`uu^@BqY+5+8 zO=*hY?x5OL(L~lZ$rEg-4Je}{yHXLL20Yl48()km1#-*rP)X(ox!8pVChAT-)}4>Y zk`%_q1hWxNr0sy061PXL=rUZ0YZ8zc1odslaJS8RjwNC-9NBW-23lE--IAh&$R+VD7Z zNEH;^G(}Oi>{zzJ#fqNq!Ia>DVoE6uk3zk!qyZyn?MT}We?kdH>s!ouGmaA%D&9ys;;K$JhFzt zCf^+x&lx2<5>K%hXxKiHHOppIH~BI`;<$y;6cIKhhNEQ8^x7!UUB@a2DfHK%PWjVH zR!M>}l^iX{6$*5tGfTvYK4i^J3jbMv1UVws*nuv^iY!fS377d!_en8Yjih0#PB5RGdQ#ZuQWo-gmt&-LXv`L(OmFpgV05%S zpv=?TcVWsfEWTtLtj|>^uQJs@?B|>r?s&voCYVb$ZFd|CL)apAtT9$+q&uu87O+x6 zYbef*Ob;<`KB{pfN%mSoYu(Oj!fnJK{NX4JD*^#=Td0}FLSb%{7VdD*+dWzYf}vgs z)&yp$A+z6u*w8HmwnA<809ykesY-Jb2+*D##Qi3w<3dg20oMrYQ7|f9K`*f{+;dj+*qI?ayA_G5SKp-@~6mA$5q7W*xwkj$Wf*maZlwhI8+wA1}) zQ}Gx+meo1g*{?!t)l;ao2079AK=IQc8m+|CNYU1u$a;`F#txKYJnXZxbPDWf+X>q| zuj`)Q8rY)0$Xawl#Kzn!ZIJl6pK!qc=^XG6>3HN5J^sqlL>r%&wLL?xTn&s?_Uk_-NA!&6^h|IjZBohq?q?x(^DiJDsuAG9QHW3`ggiS;M25rojGM8&2~$ zA>6}s*%>xLpffK@6$wqfIZaLDnx8?^r1loga>DoczT&BsLFKBvZd`a)k&|IG@yXS; zflLNuvT3>lM8u0>YqRr&%z*oP42wJzvFLUo7(B#Bi=@gnLvlH4I8xPxJr?!Wrex03 zwTHOyZg~LMgQgA8<#<};wXNy~z9_dN>);IbB{khUn+c+^AgX6eRBqL1B@ipIHzZZt zDAaMIY}Wwc=GdV&^L5hMVOMLFa+V5O%ra7rFmuTws6cd?E#||8!_2q*roY&3I_C(6x4Xs9CBwz+fh7o2$g{}3 zX^J@15)k0F&>l)ghg?Ui!-AINlq;iIwXzW}*PUVj<53S8a2yBX)N&>Ae$+}EcegBO zIabD)1$0g&(`q6Q4IZofaT(Qh+Fxe&WHcDLi`L@(nLT7$c@^s1UND!1x6B<0&ibwH zPYd1M0*%(2(W5c7+3zz@1J|ksYazmd1GBfq<~UWPbrz1&PIan-9&b6=e z<`BnB18^r7Ql}b|q{(y~ie+v_(nBF(jxB63;fv3 z#Hkh68Dtr<*QjHW5J|!vHMD9@gmlIGwJmiHhZ3n0G~jvNE=2NaxAKR1=N#1Ce*OMP zX`!~7g28TFEqICcws5A82eRbuhHFK`} zw#@-kSXH_x<`mqsDJ9JG*3h^;jENwZ)#cBL~Fm&CVWxp>CrlF?XW2b67Nm zQr(DU;nt~+$aKu5Xw%>v)?Y-;Vcfu~I{;>qTBSR-6%3JOS|4{uimB`h$akBCUzu~w+GgvHqd+ynffIDVwY}P7v$C+w zfKA$tQe4l%%0jTn>djS^o%XzuMTiM7WLtXL(HP^Efo(M}LgZ0~mE>+W6c)OZdKx_& Z +#include +#include +#include +#include +#include + +// configuration +static const bool ENABLE_TLS = true; +static const bool USE_BOTAN_TLS = true; +//static_assert(SMP_MAX_CORES > 1 || TCP_OVER_SMP == false, "SMP must be enabled"); + +struct alignas(SMP_ALIGN) HTTP_server +{ + http::Server* server = nullptr; + net::Stream::buffer_t buffer = nullptr; + net::WS_server_connector* ws_serve = nullptr; +}; +static SMP::Array httpd; + +bool accept_client(net::Socket remote, std::string origin) +{ + /* + printf("Verifying origin: \"%s\"\n" + "Verifying remote: \"%s\"\n", + origin.c_str(), remote.to_string().c_str()); + */ + (void) origin; + return remote.address() == net::ip4::Addr(10,0,0,1); +} + +void websocket_service(net::TCP& tcp, uint16_t port) +{ + if (ENABLE_TLS) + { + if (USE_BOTAN_TLS) + { + auto& filesys = fs::memdisk().fs(); + // load CA certificate + auto ca_cert = filesys.stat("/test.der"); + // load CA private key + auto ca_key = filesys.stat("/test.key"); + // load server private key + auto srv_key = filesys.stat("/server.key"); + + PER_CPU(httpd).server = new http::Botan_server( + "blabla", ca_key, ca_cert, srv_key, tcp); + } + else + { + PER_CPU(httpd).server = new http::OpenSSL_server( + "/test.pem", "/test.key", tcp); + } + } + else + { + PER_CPU(httpd).server = new http::Server(tcp); + } + + // buffer used for testing + PER_CPU(httpd).buffer = net::Stream::construct_buffer(1200); + + // Set up server connector + PER_CPU(httpd).ws_serve = new net::WS_server_connector( + [&tcp] (net::WebSocket_ptr ws) + { + assert(SMP::cpu_id() == tcp.get_cpuid()); + // sometimes we get failed WS connections + if (ws == nullptr) return; + + auto wptr = ws.release(); + // if we are still connected, attempt was verified and the handshake was accepted + assert (wptr->is_alive()); + wptr->on_read = + [] (auto message) { + printf("WebSocket on_read: %.*s\n", (int) message->size(), message->data()); + }; + wptr->on_close = + [wptr] (uint16_t) { + delete wptr; + }; + + //socket->write("THIS IS A TEST CAN YOU HEAR THIS?"); + for (int i = 0; i < 1500; i++) + wptr->write(PER_CPU(httpd).buffer, net::op_code::BINARY); + + wptr->close(); + }, + accept_client); + PER_CPU(httpd).server->on_request(*PER_CPU(httpd).ws_serve); + PER_CPU(httpd).server->listen(port); + /// server /// +} + +static void tcp_service(net::TCP& tcp) +{ + SMP_PRINT("On CPU %d with stack %p\n", SMP::cpu_id(), &tcp); + + // start a websocket server on @port + websocket_service(tcp, 8000); +} + +void Service::start() +{ + extern void create_network_device(int N, const char* route, const char* ip); + create_network_device(0, "10.0.0.0/24", "10.0.0.1"); + + auto& inet = net::Super_stack::get(0); + inet.network_config( + { 10, 0, 0, 42 }, // IP + { 255,255,255, 0 }, // Netmask + { 10, 0, 0, 1 }, // Gateway + { 10, 0, 0, 1 }); // DNS + + // echo server + inet.tcp().set_MSL(std::chrono::seconds(3)); + inet.tcp().listen(7, + [] (auto conn) + { + auto* stream = new net::tcp::Stream(conn); + + stream->on_close( + [stream] () { + printf("TCP stream on_close\n"); + delete stream; + }); + + stream->write("Hei\n"); + delete stream; + }); + + // Read-only filesystem + fs::memdisk().init_fs( + [] (auto err, auto&) { + assert(!err); + }); + + // run websocket server locally + websocket_service(inet.tcp(), 8000); +} + +#include +static void print_heap_info() +{ + const std::string heapinfo = HeapDiag::to_string(); + printf("%s\n", heapinfo.c_str()); + StackSampler::print(10); +} + +void Service::ready() +{ + //auto stats = ScopedProfiler::get_statistics(); + //printf("%.*s\n", stats.size(), stats.c_str()); + using namespace net; + + printf("Size of TCP connection: 1x %zu 1000x %zu kB\n", sizeof(tcp::Connection), (1000*sizeof(tcp::Connection))/1024); + printf("Size of TLS stream: 1x %u 1000x %u kB\n", 1024*100, 1000*100); + printf("Size of messages: 1x %u 1000x %u kB\n", 1200*1500, (1000*1500*1200)/1024); + + using namespace std::chrono; + Timers::periodic(1s, [] (int) { + //print_heap_info(); + }); + + StackSampler::begin(); +} From dfdac0747e11fc1344ce996f073b14b1de6762e6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 17:59:01 +0200 Subject: [PATCH 369/723] net: Clear headers in a more standard way --- api/net/ip4/packet_ip4.hpp | 2 +- api/net/ip6/packet_ip6.hpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/api/net/ip4/packet_ip4.hpp b/api/net/ip4/packet_ip4.hpp index 963b1297eb..4db8fd11e7 100644 --- a/api/net/ip4/packet_ip4.hpp +++ b/api/net/ip4/packet_ip4.hpp @@ -223,7 +223,7 @@ namespace net { void init(Protocol proto = Protocol::HOPOPT) noexcept { Expects(size() == 0); auto& hdr = ip_header(); - std::memset(&ip_header(), 0, sizeof(ip4::Header)); + hdr = {}; hdr.version_ihl = 0x45; //hdr.ds_ecn = 0; //hdr.id = 0; diff --git a/api/net/ip6/packet_ip6.hpp b/api/net/ip6/packet_ip6.hpp index 0131b535f3..a4068cbc7f 100644 --- a/api/net/ip6/packet_ip6.hpp +++ b/api/net/ip6/packet_ip6.hpp @@ -6,9 +6,9 @@ // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at -// +// // http://www.apache.org/licenses/LICENSE-2.0 -// +// // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -156,7 +156,7 @@ namespace net void init(Protocol proto = Protocol::HOPOPT) noexcept { Expects(size() == 0); auto& hdr = ip6_header(); - std::memset(&ip6_header(), 0, IP6_HEADER_LEN); + hdr = {}; hdr.ver_tc_fl = 0x0060; hdr.next_header = static_cast(proto); hdr.payload_length = 0x0; From 5ca2b1879ddaf10cc308a9ebd932959185918e7d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 17:59:22 +0200 Subject: [PATCH 370/723] linux: Don't override abort() on Linux platform --- src/service_name.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/service_name.cpp b/src/service_name.cpp index 8e3b6e39f8..436010f4d3 100644 --- a/src/service_name.cpp +++ b/src/service_name.cpp @@ -20,10 +20,12 @@ extern "C" __attribute__((noreturn)) void panic(const char* reason); +#ifndef __linux__ extern "C" __attribute__((noreturn)) void abort(){ panic("Abort called"); } +#endif const char* service_name__ = SERVICE_NAME; const char* service_binary_name__ = SERVICE; From d3dfdbb97e5cc1e0fe902d7738b4c00421f399f4 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 18:03:46 +0200 Subject: [PATCH 371/723] linux: Add README --- linux/README.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 linux/README.md diff --git a/linux/README.md b/linux/README.md new file mode 100644 index 0000000000..7e512922b9 --- /dev/null +++ b/linux/README.md @@ -0,0 +1,30 @@ +## Userspace Linux platform + +### Installation + +* Get a modern compiler + * gcc 7.3 or gcc 8.0 + * clang-6 or clang-7 from SVN trunk +* Install the CMake project + * mkdir -p build && cd build + * cmake .. + * make -j32 install +* Install Botan from source, if you need it + * git clone ... + * ./configure --disable-shared + * make -j32 + * sudo make install + +### Options + +* Use EXTRA_LIBS in CMake project to add your own libraries +* CMake options: + * Some options require enabling it in both the Linux platform CMake project and the service code CMake project for it to fully work. + * CUSTOM_BOTAN: Use your local Botan for Botan TLS (enable both) + * ENABLE_LTO: Enable ThinLTO. Only works with clang. (enable both) + * GPROF: Enable profiling with gprof (enable both) + * SANITIZE: Enable asan and ub sanitizers (enable both) + +### Testing it + +Run one of the Linux platform tests in /test/linux. If you get the error `RTNETLINK answers: File exists`, flush the interace with `sudo ip addr flush dev bridge43`. Where bridge43 is the interface name. From 534afd400521dcb69023091fd17ab811229392c3 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 19:16:14 +0200 Subject: [PATCH 372/723] cmake: Fix linking issue, move libc initialized to common module --- cmake/post.service.cmake | 3 +++ src/kernel/os.cpp | 4 ++-- src/platform/x86_nano/kernel_start.cpp | 3 --- src/platform/x86_pc/kernel_start.cpp | 2 +- src/platform/x86_solo5/platform.cpp | 3 --- test/lest_util/os_mock.cpp | 3 --- 6 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index bad2632b0f..cf6cdd649b 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -450,6 +450,9 @@ endif() # all the OS and C/C++ libraries + crt end target_link_libraries(service libos + libplatform + libarch + ${LIBR_CMAKE_NAMES} libos libbotan diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 74508e9630..389e5c1407 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -48,7 +48,8 @@ extern uintptr_t _TEXT_START_; extern uintptr_t _LOAD_START_; extern uintptr_t _ELF_END_; -// Initialize static OS data members +bool __libc_initialized = false; + bool OS::power_ = true; bool OS::boot_sequence_passed_ = false; bool OS::m_is_live_updated = false; @@ -180,7 +181,6 @@ __attribute__((weak)) bool os_enable_boot_logging = false; __attribute__((weak)) bool os_default_stdout = false; -extern bool __libc_initialized; void OS::print(const char* str, const size_t len) { diff --git a/src/platform/x86_nano/kernel_start.cpp b/src/platform/x86_nano/kernel_start.cpp index 905a8aab3c..92cea8cb2a 100644 --- a/src/platform/x86_nano/kernel_start.cpp +++ b/src/platform/x86_nano/kernel_start.cpp @@ -32,9 +32,6 @@ extern "C" { uintptr_t _multiboot_free_begin(uintptr_t boot_addr); uintptr_t _multiboot_memory_end(uintptr_t boot_addr); - - -bool __libc_initialized = true; extern bool os_default_stdout; extern "C" diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 870f61e9cc..2e39935b45 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -61,7 +61,6 @@ uint32_t __grub_magic = 0xc001; uint32_t __grub_addr = 0x7001; static volatile int __global_ctors_ok = 0; -bool __libc_initialized = false; void _init_bss() { @@ -77,6 +76,7 @@ static void global_ctor_test(){ int kernel_main(int, char * *, char * *) { PRATTLE(" libc initialization complete \n"); Expects(__global_ctors_ok == 42); + extern bool __libc_initialized; __libc_initialized = true; Expects(__tl1__ == 42); diff --git a/src/platform/x86_solo5/platform.cpp b/src/platform/x86_solo5/platform.cpp index e93188d2d0..4d80081cc8 100644 --- a/src/platform/x86_solo5/platform.cpp +++ b/src/platform/x86_solo5/platform.cpp @@ -1,9 +1,6 @@ #include #include -bool __libc_initialized = false; - - void __arch_poweroff() { while (true) asm ("cli; hlt"); diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index 554c958b84..1c4855ce2b 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -112,9 +112,6 @@ char __plugin_ctors_start; char __plugin_ctors_end; char __service_ctors_start; char __service_ctors_end; -bool __libc_initialized = true; - - char _ELF_START_; char _ELF_END_; From 93cbf26112e74259b23fe991860528bf1a4f75d6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 20:45:56 +0200 Subject: [PATCH 373/723] net: Initialize IP4/IP6 headers by defaults --- api/net/ip4/header.hpp | 10 +++++----- api/net/ip4/packet_ip4.hpp | 9 ++------- api/net/ip6/header.hpp | 8 ++++---- api/net/ip6/packet_ip6.hpp | 6 +++--- test/net/unit/ip6.cpp | 2 +- 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/api/net/ip4/header.hpp b/api/net/ip4/header.hpp index 5c17b7e980..ac1f41d5fd 100644 --- a/api/net/ip4/header.hpp +++ b/api/net/ip4/header.hpp @@ -39,11 +39,11 @@ enum class Flags : uint8_t { * This type is used to represent the standard IPv4 header */ struct Header { - uint8_t version_ihl; //< IP version and header length - uint8_t ds_ecn; //< IP datagram differentiated services codepoint and explicit congestion notification - uint16_t tot_len; //< IP datagram total length - uint16_t id; //< IP datagram identification number - uint16_t frag_off_flags; //< IP datagram fragment offset and control flags + uint8_t version_ihl = 0x45; //< IP version and header length + uint8_t ds_ecn = 0; //< IP datagram differentiated services codepoint and explicit congestion notification + uint16_t tot_len = 0; //< IP datagram total length + uint16_t id = 0; //< IP datagram identification number + uint16_t frag_off_flags = 0; //< IP datagram fragment offset and control flags uint8_t ttl; //< IP datagram time to live value uint8_t protocol; //< IP datagram protocol value uint16_t check; //< IP datagram checksum value diff --git a/api/net/ip4/packet_ip4.hpp b/api/net/ip4/packet_ip4.hpp index 4db8fd11e7..69c51914a5 100644 --- a/api/net/ip4/packet_ip4.hpp +++ b/api/net/ip4/packet_ip4.hpp @@ -28,7 +28,7 @@ namespace net { /** IPv4 packet. */ class PacketIP4 : public Packet { public: - static constexpr size_t DEFAULT_TTL {64}; + static constexpr int DEFAULT_TTL = 64; using Span = gsl::span; using Cspan = gsl::span; @@ -224,14 +224,9 @@ namespace net { Expects(size() == 0); auto& hdr = ip_header(); hdr = {}; - hdr.version_ihl = 0x45; - //hdr.ds_ecn = 0; - //hdr.id = 0; - //hdr.frag_off_flags = 0; + hdr.tot_len = 0x1400; // Big-endian 20 hdr.ttl = DEFAULT_TTL; hdr.protocol = static_cast(proto); - //hdr.check = 0; - hdr.tot_len = 0x1400; // Big-endian 20 increment_data_end(sizeof(ip4::Header)); } diff --git a/api/net/ip6/header.hpp b/api/net/ip6/header.hpp index 69f37b0965..77cce77579 100644 --- a/api/net/ip6/header.hpp +++ b/api/net/ip6/header.hpp @@ -31,14 +31,14 @@ namespace ip6 { */ struct Header { union { - uint32_t ver_tc_fl; + uint32_t ver_tc_fl = 0x0060; uint32_t version : 4, traffic_class : 8, flow_label : 20; }; - uint16_t payload_length; - uint8_t next_header; - uint8_t hop_limit; + uint16_t payload_length = 0; + uint8_t next_header = 0; + uint8_t hop_limit = 0; Addr saddr; Addr daddr; }; //< struct Header diff --git a/api/net/ip6/packet_ip6.hpp b/api/net/ip6/packet_ip6.hpp index a4068cbc7f..6906b6eee9 100644 --- a/api/net/ip6/packet_ip6.hpp +++ b/api/net/ip6/packet_ip6.hpp @@ -29,6 +29,7 @@ namespace net /** IPv6 packet. */ class PacketIP6 : public Packet { public: + static constexpr int DEFAULT_TTL = 64; using Span = gsl::span; using Cspan = gsl::span; @@ -157,9 +158,8 @@ namespace net Expects(size() == 0); auto& hdr = ip6_header(); hdr = {}; - hdr.ver_tc_fl = 0x0060; - hdr.next_header = static_cast(proto); - hdr.payload_length = 0x0; + hdr.hop_limit = DEFAULT_TTL; + hdr.next_header = static_cast(proto); increment_data_end(IP6_HEADER_LEN); } diff --git a/test/net/unit/ip6.cpp b/test/net/unit/ip6.cpp index c34e65cc2a..778a399fc2 100644 --- a/test/net/unit/ip6.cpp +++ b/test/net/unit/ip6.cpp @@ -39,7 +39,7 @@ CASE("IP6 packet setters/getters") EXPECT(ip_pckt1->flow_label() == 0); EXPECT(ip_pckt1->payload_length() == 0); EXPECT(ip_pckt1->next_protocol() == Protocol::ICMPv4); - EXPECT(ip_pckt1->hop_limit() == 0); + EXPECT(ip_pckt1->hop_limit() == PacketIP6::DEFAULT_TTL); EXPECT(ip_pckt1->ip_src() == IP6::ADDR_ANY); EXPECT(ip_pckt1->ip_dst() == IP6::ADDR_ANY); } From c4b2fd41f8325cd52c005d9e16bcf7c1d697d68f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 21:13:25 +0200 Subject: [PATCH 374/723] util: Fix uninitialized storage_ warning in delegate --- api/util/delegate.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/util/delegate.hpp b/api/util/delegate.hpp index d8d469cfed..3087322b4e 100644 --- a/api/util/delegate.hpp +++ b/api/util/delegate.hpp @@ -338,7 +338,7 @@ template< } private: - mutable storage_t storage_; + mutable storage_t storage_ {}; invoke_ptr_t invoke_ptr_; copy_ptr_t copy_ptr_; From 794a7e6eff1a2a382565a4325afaf74aea888ce8 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 21:14:02 +0200 Subject: [PATCH 375/723] linux: Fix warning, fix Botan 2 linking on GCC --- cmake/linux.service.cmake | 6 +++++- linux/src/os.cpp | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index c20eb8299f..5753a73a04 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -74,10 +74,14 @@ add_executable(service ${SOURCES} ${IOSPATH}/src/service_name.cpp) set_target_properties(service PROPERTIES OUTPUT_NAME ${BINARY}) if (CUSTOM_BOTAN) - target_link_libraries(service /usr/local/lib/libbotan-2.a -ldl -pthread) + set(BOTAN_LIBS /usr/local/lib/libbotan-2.a) + target_link_libraries(service ${BOTAN_LIBS} -ldl -pthread) endif() target_link_libraries(service ${EXTRA_LIBS}) target_link_libraries(service includeos linuxrt includeos http_parser rt) +if (CUSTOM_BOTAN) + target_link_libraries(service ${BOTAN_LIBS}) +endif() if (STATIC_BUILD) target_link_libraries(service -static-libstdc++ -static-libgcc) diff --git a/linux/src/os.cpp b/linux/src/os.cpp index 37a8e6a3df..9d359e4388 100644 --- a/linux/src/os.cpp +++ b/linux/src/os.cpp @@ -104,7 +104,8 @@ void OS::start(char* cmdline, uintptr_t) // stdout void OS::default_stdout(const char* text, size_t len) { - write(STDOUT_FILENO, text, len); + ssize_t bytes = write(STDOUT_FILENO, text, len); + assert(bytes == (ssize_t) len); } // system_log has no place on Linux because stdout goes --> pipe From a020d3b4e88e95d7fa111e723ab374e6b9007baa Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 21:14:52 +0200 Subject: [PATCH 376/723] openssl: Don't close when busy on callback_once --- api/net/openssl/tls_stream.hpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/api/net/openssl/tls_stream.hpp b/api/net/openssl/tls_stream.hpp index 2f183e58c5..6ff53ada1a 100644 --- a/api/net/openssl/tls_stream.hpp +++ b/api/net/openssl/tls_stream.hpp @@ -311,9 +311,9 @@ namespace openssl inline void TLS_stream::close() { - ERR_clear_error(); + //ERR_clear_error(); if (this->m_busy) { - m_deferred_close = true; return; + this->m_deferred_close = true; return; } CloseCallback func = std::move(this->m_on_close); this->reset_callbacks(); @@ -323,6 +323,9 @@ namespace openssl } inline void TLS_stream::close_callback_once() { + if (this->m_busy) { + this->m_deferred_close = true; return; + } CloseCallback func = std::move(this->m_on_close); this->reset_callbacks(); if (func) func(); From ecc9d46086c0f381de94c35b0ed0f46d8d98597e Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 21:17:16 +0200 Subject: [PATCH 377/723] icmp4: Avoid fallthrough on DEST_UNREACHABLE (bug) --- src/net/ip4/icmp4.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/net/ip4/icmp4.cpp b/src/net/ip4/icmp4.cpp index 09eff205d3..a11b4a5cde 100644 --- a/src/net/ip4/icmp4.cpp +++ b/src/net/ip4/icmp4.cpp @@ -53,33 +53,33 @@ namespace net { auto req = icmp4::Packet(std::move(pckt_ip4)); switch(req.type()) { - case (ICMP_type::ECHO): + case ICMP_type::ECHO: debug(" PING from %s\n", req.ip().ip_src().to_string().c_str()); ping_reply(req); break; - case (ICMP_type::ECHO_REPLY): + case ICMP_type::ECHO_REPLY: debug(" PING Reply from %s\n", req.ip().ip_src().str().c_str()); execute_ping_callback(req); break; - case (ICMP_type::DEST_UNREACHABLE): + case ICMP_type::DEST_UNREACHABLE: if (req.code() == (uint8_t) icmp4::code::Dest_unreachable::FRAGMENTATION_NEEDED) { debug(" ICMP Too Big message from %s\n", req.ip().ip_src().str().c_str()); handle_too_big(req); return; } - // else continue - case (ICMP_type::REDIRECT): - case (ICMP_type::TIME_EXCEEDED): - case (ICMP_type::PARAMETER_PROBLEM): + break; // else continue + case ICMP_type::REDIRECT: + case ICMP_type::TIME_EXCEEDED: + case ICMP_type::PARAMETER_PROBLEM: debug(" ICMP error message from %s\n", req.ip().ip_src().str().c_str()); forward_to_transport_layer(req); break; - case (ICMP_type::TIMESTAMP): + case ICMP_type::TIMESTAMP: debug(" TIMESTAMP from %s\n", req.ip().ip_src().str().c_str()); // TODO May // timestamp_reply(req); break; - case (ICMP_type::TIMESTAMP_REPLY): + case ICMP_type::TIMESTAMP_REPLY: debug(" TIMESTAMP REPLY from %s\n", req.ip().ip_src().str().c_str()); // TODO May break; From 1bfd647f83cc3917545d0d2e2e8aae5cec76befe Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 21:21:38 +0200 Subject: [PATCH 378/723] cpuid: Fix possible unhandled return paths --- src/kernel/cpuid.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kernel/cpuid.cpp b/src/kernel/cpuid.cpp index 4c45aea616..9e843a8234 100644 --- a/src/kernel/cpuid.cpp +++ b/src/kernel/cpuid.cpp @@ -211,8 +211,7 @@ namespace case Feature::BMI1: return FeatureInfo { 7, 0, Register::ECX, 1u << 3 }; // BMI1 case Feature::BMI2: return FeatureInfo { 7, 0, Register::ECX, 1u << 8 }; // BMI2 case Feature::LZCNT: return FeatureInfo { 7, 0, Register::ECX, 1u << 5 }; // LZCNT - - + default: throw std::out_of_range("Unimplemented CPU feature encountered"); } } @@ -308,6 +307,7 @@ bool CPUID::has_feature(Feature f) case Register::ECX: return (cpuid_result.ECX & feature_info.bitmask) != 0; case Register::EDX: return (cpuid_result.EDX & feature_info.bitmask) != 0; } + return false; } #define KVM_CPUID_SIGNATURE 0x40000000 From 495eef95be67f69cc60613a26b66b9f1fc58b1f0 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 21:43:23 +0200 Subject: [PATCH 379/723] musl: Remove duplicate tkill --- src/musl/CMakeLists.txt | 2 +- src/musl/kill.cpp | 3 +++ src/musl/tkill.cpp | 15 --------------- 3 files changed, 4 insertions(+), 16 deletions(-) delete mode 100644 src/musl/tkill.cpp diff --git a/src/musl/CMakeLists.txt b/src/musl/CMakeLists.txt index 7f4ff26637..123edd43df 100644 --- a/src/musl/CMakeLists.txt +++ b/src/musl/CMakeLists.txt @@ -7,7 +7,7 @@ set(MUSL_OBJECTS nanosleep.cpp open.cpp creat.cpp clock_gettime.cpp gettimeofday.cpp poll.cpp exit.cpp close.cpp set_tid_address.cpp pipe.cpp read.cpp readv.cpp getpid.cpp getuid.cpp mknod.cpp sync.cpp - msync.cpp mincore.cpp syscall_n.cpp sigmask.cpp gettid.cpp tkill.cpp + msync.cpp mincore.cpp syscall_n.cpp sigmask.cpp gettid.cpp socketcall.cpp rt_sigaction.cpp stat.cpp fstat.cpp fstatat.cpp access.cpp diff --git a/src/musl/kill.cpp b/src/musl/kill.cpp index 5c4e10bd1e..bd30feb893 100644 --- a/src/musl/kill.cpp +++ b/src/musl/kill.cpp @@ -6,6 +6,9 @@ int sys_kill(pid_t /*pid*/, int /*sig*/) { } int sys_tkill(int /*tid*/, int /*sig*/) { +#ifndef INCLUDEOS_SINGLE_THREADED +# warning "tkill not implemented for threaded IncludeOS" +#endif panic("TKILL called"); } diff --git a/src/musl/tkill.cpp b/src/musl/tkill.cpp deleted file mode 100644 index 0ecba0b550..0000000000 --- a/src/musl/tkill.cpp +++ /dev/null @@ -1,15 +0,0 @@ -#include "stub.hpp" -#include - -static int sys_tkill(int /*tid*/, int sig) -{ -#ifndef INCLUDEOS_SINGLE_THREADED -# warning "tkill not implemented for threaded IncludeOS" -#endif - exit(sig); -} - -extern "C" -int syscall_SYS_tkill(int tid, int sig) { - return stubtrace(sys_tkill, "tkill", tid, sig); -} From 66b8681fac22d9a23eb945df60481f5423a89154 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 22:10:35 +0200 Subject: [PATCH 380/723] linux: Remove __libc_initialized from linux platform also --- linux/src/os.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/linux/src/os.cpp b/linux/src/os.cpp index 9d359e4388..1645de87ec 100644 --- a/linux/src/os.cpp +++ b/linux/src/os.cpp @@ -6,7 +6,7 @@ #include #include // mallinfo() #include -bool __libc_initialized = true; +extern bool __libc_initialized; void OS::event_loop() { @@ -83,6 +83,7 @@ static void stop_timers() {} #include void OS::start(char* cmdline, uintptr_t) { + __libc_initialized = true; // statman static char statman_data[1 << 16]; Statman::get().init((uintptr_t) statman_data, sizeof(statman_data)); From 6330f1a2b268b8fcfd50095478ad29f5d6e13e1f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 1 Jun 2018 22:10:58 +0200 Subject: [PATCH 381/723] ip6: Rename DEFAULT_TTL to DEFAULT_HOP_LIMIT --- api/net/ip6/packet_ip6.hpp | 4 ++-- test/net/unit/ip6.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/net/ip6/packet_ip6.hpp b/api/net/ip6/packet_ip6.hpp index 6906b6eee9..04d3ee0319 100644 --- a/api/net/ip6/packet_ip6.hpp +++ b/api/net/ip6/packet_ip6.hpp @@ -29,7 +29,7 @@ namespace net /** IPv6 packet. */ class PacketIP6 : public Packet { public: - static constexpr int DEFAULT_TTL = 64; + static constexpr int DEFAULT_HOP_LIMIT = 64; using Span = gsl::span; using Cspan = gsl::span; @@ -158,7 +158,7 @@ namespace net Expects(size() == 0); auto& hdr = ip6_header(); hdr = {}; - hdr.hop_limit = DEFAULT_TTL; + hdr.hop_limit = DEFAULT_HOP_LIMIT; hdr.next_header = static_cast(proto); increment_data_end(IP6_HEADER_LEN); } diff --git a/test/net/unit/ip6.cpp b/test/net/unit/ip6.cpp index 778a399fc2..de480c0a89 100644 --- a/test/net/unit/ip6.cpp +++ b/test/net/unit/ip6.cpp @@ -39,7 +39,7 @@ CASE("IP6 packet setters/getters") EXPECT(ip_pckt1->flow_label() == 0); EXPECT(ip_pckt1->payload_length() == 0); EXPECT(ip_pckt1->next_protocol() == Protocol::ICMPv4); - EXPECT(ip_pckt1->hop_limit() == PacketIP6::DEFAULT_TTL); + EXPECT(ip_pckt1->hop_limit() == PacketIP6::DEFAULT_HOP_LIMIT); EXPECT(ip_pckt1->ip_src() == IP6::ADDR_ANY); EXPECT(ip_pckt1->ip_dst() == IP6::ADDR_ANY); } From e5a4da6b806efce1ded3a2d8bba8fccbeb582c59 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Sat, 2 Jun 2018 11:44:15 +0200 Subject: [PATCH 382/723] linux: Enough to run websockets on GCC 8 --- cmake/linux.service.cmake | 2 +- linux/src/drivers/memdisk.cpp | 19 +++++++++---------- linux/src/profile.cpp | 5 +++++ src/net/https/openssl_server.cpp | 2 +- test/linux/websockets/run.sh | 7 ++++--- test/linux/websockets/service.cpp | 4 ++-- 6 files changed, 22 insertions(+), 17 deletions(-) diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index 5753a73a04..bdd80587ff 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -77,8 +77,8 @@ if (CUSTOM_BOTAN) set(BOTAN_LIBS /usr/local/lib/libbotan-2.a) target_link_libraries(service ${BOTAN_LIBS} -ldl -pthread) endif() -target_link_libraries(service ${EXTRA_LIBS}) target_link_libraries(service includeos linuxrt includeos http_parser rt) +target_link_libraries(service ${EXTRA_LIBS}) if (CUSTOM_BOTAN) target_link_libraries(service ${BOTAN_LIBS}) endif() diff --git a/linux/src/drivers/memdisk.cpp b/linux/src/drivers/memdisk.cpp index 019e219627..b09042cf34 100644 --- a/linux/src/drivers/memdisk.cpp +++ b/linux/src/drivers/memdisk.cpp @@ -6,19 +6,18 @@ namespace fs { static uint64_t stat; MemDisk::MemDisk() noexcept - : image_start_(0), - image_end_(0), - stat_read(stat) + : stat_read(stat) { auto* fp = fopen("memdisk.fat", "rb"); assert(fp != nullptr && "Open memdisk.fat in source dir"); fseek(fp, 0L, SEEK_END); - int size = ftell(fp); + long int size = ftell(fp); rewind(fp); + //printf("MemDisk::MemDisk %zd size -> %zd sectors\n", size, size / SECTOR_SIZE); // read file into buffer char* buffer = new char[size]; - int res = fread(buffer, sizeof(char), size, fp); - assert(res == size); + size_t res = fread(buffer, size, 1, fp); + assert(res == 1); // set image start/end image_start_ = buffer; image_end_ = buffer + size; @@ -26,7 +25,7 @@ MemDisk::MemDisk() noexcept MemDisk::block_t MemDisk::size() const noexcept { - return image_end_ - image_start_; + return (image_end_ - image_start_) / SECTOR_SIZE; } buffer_t MemDisk::read_sync(block_t blk) @@ -35,10 +34,10 @@ buffer_t MemDisk::read_sync(block_t blk) } buffer_t MemDisk::read_sync(block_t blk, size_t cnt) { - if ((blk + cnt) * SECTOR_SIZE > size()) return nullptr; + //printf("MemDisk::read %zu -> %zu / %zu\n", blk, blk + cnt, size() / SECTOR_SIZE); + if (blk + cnt > size()) return nullptr; auto* start = &image_start_[blk * SECTOR_SIZE]; - auto* end = start + cnt * SECTOR_SIZE; - return fs::construct_buffer(start, end); + return fs::construct_buffer(start, start + cnt * SECTOR_SIZE); } void MemDisk::deactivate() { diff --git a/linux/src/profile.cpp b/linux/src/profile.cpp index dab9576536..d27e4a7f6c 100644 --- a/linux/src/profile.cpp +++ b/linux/src/profile.cpp @@ -9,3 +9,8 @@ uint64_t StackSampler::samples_asleep() noexcept { } void StackSampler::print(int) {} void StackSampler::set_mode(mode_t) {} + +std::string HeapDiag::to_string() +{ + return ""; +} diff --git a/src/net/https/openssl_server.cpp b/src/net/https/openssl_server.cpp index c631d88dd9..c683ffd699 100644 --- a/src/net/https/openssl_server.cpp +++ b/src/net/https/openssl_server.cpp @@ -9,7 +9,7 @@ namespace http const std::string& key) { fs::memdisk().init_fs( - [] (auto err, auto&) { + [] (fs::error_t err, fs::File_system&) { assert(!err); }); /** INIT OPENSSL **/ diff --git a/test/linux/websockets/run.sh b/test/linux/websockets/run.sh index b7a335614e..e345227f87 100755 --- a/test/linux/websockets/run.sh +++ b/test/linux/websockets/run.sh @@ -3,12 +3,13 @@ set -e export GPROF="OFF" +export DBG="OFF" -export num_jobs=${num_jobs:--j4} +export num_jobs=${num_jobs:--j32} function make_linux(){ pushd $INCLUDEOS_SRC/linux/build - cmake -DGPROF=$GPROF .. + cmake -DGPROF=$GPROF -DDEBUGGING=$DBG .. make $num_jobs install popd } @@ -16,7 +17,7 @@ function make_linux(){ function make_service(){ mkdir -p build pushd build - cmake -DGPROF=$GPROF .. + cmake -DGPROF=$GPROF -DDEBUGGING=$DBG .. make $num_jobs popd } diff --git a/test/linux/websockets/service.cpp b/test/linux/websockets/service.cpp index b94b60f8e3..86e33fadd0 100644 --- a/test/linux/websockets/service.cpp +++ b/test/linux/websockets/service.cpp @@ -7,7 +7,7 @@ // configuration static const bool ENABLE_TLS = true; -static const bool USE_BOTAN_TLS = true; +static const bool USE_BOTAN_TLS = false; //static_assert(SMP_MAX_CORES > 1 || TCP_OVER_SMP == false, "SMP must be enabled"); struct alignas(SMP_ALIGN) HTTP_server @@ -131,7 +131,7 @@ void Service::start() // Read-only filesystem fs::memdisk().init_fs( - [] (auto err, auto&) { + [] (fs::error_t err, fs::File_system&) { assert(!err); }); From 50694e2be2f90cd3cc1cf00513923724e10d3aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 4 Jun 2018 11:30:26 +0200 Subject: [PATCH 383/723] net: Add union class to handle both IP address v4 and v6 --- api/net/addr.hpp | 124 ++++++++++++++++++++++++++++++++++++ test/net/unit/addr_test.cpp | 37 +++++++++++ 2 files changed, 161 insertions(+) create mode 100644 api/net/addr.hpp create mode 100644 test/net/unit/addr_test.cpp diff --git a/api/net/addr.hpp b/api/net/addr.hpp new file mode 100644 index 0000000000..52c10e9751 --- /dev/null +++ b/api/net/addr.hpp @@ -0,0 +1,124 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include + +namespace net { + +union Addr { +public: + static const Addr addr_any; + + constexpr Addr() noexcept + : ip6_{} {} + + Addr(ip4::Addr addr) noexcept + : ip4_{0, ip4_signature, std::move(addr)} {} + + Addr(ip6::Addr addr) noexcept + : ip6_{std::move(addr)} {} + + bool is_v4() const noexcept + { return ip4_.big == 0 and ip4_.sign == ip4_signature; } + + bool is_v6() const noexcept + { return not is_v4(); } + + void set_v4(ip4::Addr addr) noexcept + { + ip4_.big = 0; + ip4_.sign = ip4_signature; + ip4_.addr = std::move(addr); + } + + void set_v6(ip6::Addr addr) noexcept + { + ip6_ = std::move(addr); + } + + ip4::Addr v4() const + { + if(UNLIKELY(not is_v4())) + throw ip4::Invalid_address{"Address is not v4"}; + return ip4_.addr; + } + + const ip6::Addr& v6() const noexcept + { return ip6_; } + + bool is_any() const noexcept + { return (is_v4() and ip4_.addr == 0) or ip6_ == ip6::Addr::addr_any; } + + Addr any_addr() const noexcept + { return is_v4() ? Addr{ip4::Addr::addr_any} : Addr{ip6::Addr::addr_any}; } + + std::string to_string() const + { return is_v4() ? ip4_.addr.to_string() : ip6_.to_string(); } + + Addr(const Addr& other) noexcept + : ip6_{other.ip6_} {} + + Addr(Addr&& other) noexcept + : ip6_{std::move(other.ip6_)} {} + + Addr& operator=(const Addr& other) noexcept + { + ip6_ = other.ip6_; + return *this; + } + Addr& operator=(Addr&& other) noexcept + { + ip6_ = std::move(other.ip6_); + return *this; + } + + //operator ip4::Addr() const + //{ return v4(); } + + //operator const ip6::Addr&() const + //{ return v6(); } + + bool operator==(const Addr& other) const noexcept + { return ip6_ == other.ip6_; } + + bool operator>(const Addr& other) const noexcept + { return ip6_ > other.ip6_; } + + bool operator>=(const Addr& other) const noexcept + { return (*this > other or *this == other); } + + bool operator<(const Addr& other) const noexcept + { return not (*this >= other); } + + bool operator<=(const Addr& other) const noexcept + { return (*this < other or *this == other); } + +private: + struct { + uint64_t big; + uint32_t sign; + ip4::Addr addr; + } ip4_; + ip6::Addr ip6_; + + static constexpr uint32_t ip4_signature{0x0000FFFF}; +}; + +static_assert(sizeof(Addr) == sizeof(ip6::Addr)); + +} diff --git a/test/net/unit/addr_test.cpp b/test/net/unit/addr_test.cpp new file mode 100644 index 0000000000..e9c144a9cc --- /dev/null +++ b/test/net/unit/addr_test.cpp @@ -0,0 +1,37 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2016 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +using namespace net; + +CASE("Addr") +{ + Addr addr{}; + + ip4::Addr ip4addr; + ip6::Addr ip6addr; + + //EXPECT(addr.v4() == ip4addr); + EXPECT(addr.v6() == ip6addr); + + addr.set_v4({10,0,0,42}); + printf("%s\n", addr.v4().to_string().c_str()); + printf("%s\n", addr.v6().to_string().c_str()); + +} From 0cca2151fedec4d804ea60d070dabed5cba005be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 4 Jun 2018 11:37:49 +0200 Subject: [PATCH 384/723] net: IP6 integration, major changes to address handling. Socket is no bound to IP4, started on TCP support for new socket --- api/net/dns/client.hpp | 4 +- api/net/inet.hpp | 13 +++--- api/net/ip4/addr.hpp | 6 +++ api/net/ip4/ip4.hpp | 1 - api/net/ip4/udp.hpp | 6 +-- api/net/ip4/udp_socket.hpp | 4 +- api/net/ip6/addr.hpp | 16 ++++++- api/net/socket.hpp | 71 ++++++++++++++++-------------- api/net/tcp/common.hpp | 4 +- api/net/tcp/listener.hpp | 4 +- api/net/tcp/packet.hpp | 4 +- api/net/tcp/tcp.hpp | 28 ++++-------- api/util/syslog_facility.hpp | 4 +- lib/mender/src/client.cpp | 2 +- lib/microLB/micro_lb/autoconf.cpp | 2 +- src/net/dhcp/dh4client.cpp | 4 +- src/net/dhcp/dhcpd.cpp | 2 +- src/net/dns/client.cpp | 4 +- src/net/ip4/ip4.cpp | 3 +- src/net/ip4/udp.cpp | 8 ++-- src/net/ip4/udp_socket.cpp | 4 +- src/net/ip6/ip6.cpp | 1 + src/net/nat/napt.cpp | 12 ++--- src/net/nat/nat.cpp | 12 ++--- src/net/tcp/listener.cpp | 6 ++- src/net/tcp/tcp.cpp | 73 +++++++++++++++++++++++-------- src/plugins/unik.cpp | 4 +- src/posix/tcp_fd.cpp | 2 +- src/posix/udp_fd.cpp | 6 +-- 29 files changed, 185 insertions(+), 125 deletions(-) diff --git a/api/net/dns/client.hpp b/api/net/dns/client.hpp index 4fbb5d99ef..c7f41cb35d 100644 --- a/api/net/dns/client.hpp +++ b/api/net/dns/client.hpp @@ -38,7 +38,7 @@ namespace net public: using Stack = IP4::Stack; using Resolve_handler = IP4::resolve_func; - using Address = ip4::Addr; + using Address = net::Addr; using Hostname = std::string; using timestamp_t = RTC::timestamp_t; /** @@ -164,7 +164,7 @@ namespace net * @param[in] data The raw data containing the msg * @param[in] Size of the data */ - void receive_response(IP4::addr, UDP::port_t, const char* data, size_t); + void receive_response(Address, UDP::port_t, const char* data, size_t); /** * @brief Adds a cache entry. diff --git a/api/net/inet.hpp b/api/net/inet.hpp index 8c9de420fa..803f73945b 100644 --- a/api/net/inet.hpp +++ b/api/net/inet.hpp @@ -60,7 +60,7 @@ namespace net { using on_configured_func = delegate; using dhcp_timeout_func = delegate; - using Port_utils = std::map; + using Port_utils = std::map; using Vip4_list = std::vector; using Vip6_list = std::vector; @@ -360,14 +360,14 @@ namespace net { { return vip6s_; } /** Check if IP4 address is virtual loopback */ - bool is_loopback(IP4::addr a) + bool is_loopback(IP4::addr a) const { return a.is_loopback() or std::find( vip4s_.begin(), vip4s_.end(), a) != vip4s_.end(); } /** Check if IP6 address is virtual loopback */ - bool is_loopback(IP6::addr a) + bool is_loopback(IP6::addr a) const { return a.is_loopback() or std::find( vip6s_.begin(), vip6s_.end(), a) != vip6s_.end(); @@ -429,10 +429,13 @@ namespace net { return ip6_addr(); } - bool is_valid_source(IP4::addr src) + bool is_valid_source(const Addr& addr) const + { return addr.is_v4() ? is_valid_source4(addr.v4()) : is_valid_source6(addr.v6()); } + + bool is_valid_source4(IP4::addr src) const { return src == ip_addr() or is_loopback(src); } - bool is_valid_source(IP6::addr src) + bool is_valid_source6(const IP6::addr& src) const { return src == ip6_addr() or is_loopback(src) or src.is_multicast(); } std::shared_ptr& conntrack() diff --git a/api/net/ip4/addr.hpp b/api/net/ip4/addr.hpp index 22b4a20e33..3c92e120ec 100644 --- a/api/net/ip4/addr.hpp +++ b/api/net/ip4/addr.hpp @@ -381,11 +381,16 @@ struct Addr { bool is_multicast() const noexcept { return part(3) >= 224 and part(3) < 240; } + static const Addr addr_any; + /* Data member */ uint32_t whole; } __attribute__((packed)); //< struct Addr +static_assert(sizeof(Addr) == 4); + } //< namespace ip4 + } //< namespace net // Allow an IPv4 address to be used as key in e.g. std::unordered_map @@ -398,4 +403,5 @@ namespace std { }; } //< namespace std + #endif //< NET_IP4_ADDR_HPP diff --git a/api/net/ip4/ip4.hpp b/api/net/ip4/ip4.hpp index 332a53413d..af34a82276 100644 --- a/api/net/ip4/ip4.hpp +++ b/api/net/ip4/ip4.hpp @@ -68,7 +68,6 @@ namespace net { using drop_handler = delegate; using Forward_delg = delegate; using PMTU = uint16_t; - using Port_utils = std::map; using resolve_func = delegate; /** Initialize. Sets a dummy linklayer out. */ diff --git a/api/net/ip4/udp.hpp b/api/net/ip4/udp.hpp index a2cb22404e..24e10c0fea 100644 --- a/api/net/ip4/udp.hpp +++ b/api/net/ip4/udp.hpp @@ -43,12 +43,12 @@ namespace net { /** Basic UDP support. @todo Implement UDP sockets. */ class UDP { public: - using addr_t = IP4::addr; + using addr_t = net::Addr; using port_t = uint16_t; using Packet_ptr = std::unique_ptr>; using Stack = IP4::Stack; - using Port_utils = IP4::Port_utils; + using Port_utils = std::map; using Sockets = std::map; @@ -130,7 +130,7 @@ namespace net { //! returns a new UDP socket bound to a random port UDPSocket& bind(); - UDPSocket& bind(const ip4::Addr addr); + UDPSocket& bind(const addr_t addr); bool is_bound(const Socket) const; bool is_bound(const port_t port) const; diff --git a/api/net/ip4/udp_socket.hpp b/api/net/ip4/udp_socket.hpp index 37f8837b82..baf8ec6f83 100644 --- a/api/net/ip4/udp_socket.hpp +++ b/api/net/ip4/udp_socket.hpp @@ -28,7 +28,7 @@ namespace net public: typedef UDP::port_t port_t; - typedef IP4::addr addr_t; + typedef net::Addr addr_t; typedef IP4::addr multicast_group_addr; typedef delegate recvfrom_handler; @@ -65,7 +65,7 @@ namespace net // stuff addr_t local_addr() const - { return socket_.address(); } + { return socket_.address().v4(); } port_t local_port() const { return socket_.port(); } diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index aeb21ec3b1..8f1067f51b 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -40,7 +40,7 @@ struct Invalid_Address : public std::runtime_error { * IPv6 Address representation */ struct Addr { - Addr() + constexpr Addr() noexcept : i32{{0, 0, 0, 0}} {} Addr(uint16_t a1, uint16_t a2, uint16_t b1, uint16_t b2, @@ -89,6 +89,7 @@ struct Addr { // unspecified link-local Address static const Addr link_unspecified; + static const Addr addr_any; // RFC 4291 2.4.6: // Link-Local Addresses are designed to be used for Addressing on a @@ -226,6 +227,19 @@ struct Addr { std::array i16; }; } __attribute__((packed)); //< struct Addr +static_assert(sizeof(Addr) == 16); + } //< namespace ip6 } //< namespace net + +// Allow an IPv6 address to be used as key in e.g. std::unordered_map +namespace std { + template<> + struct hash { + size_t operator()(const net::ip6::Addr& addr) const { + // This is temporary. Use a proper hash + return std::hash{}(addr.i64[0] + addr.i64[1]); + } + }; +} //< namespace std #endif diff --git a/api/net/socket.hpp b/api/net/socket.hpp index d8a0943171..228ff3dd2f 100644 --- a/api/net/socket.hpp +++ b/api/net/socket.hpp @@ -19,16 +19,16 @@ #ifndef NET_SOCKET_HPP #define NET_SOCKET_HPP -#include +#include namespace net { /** * An IP address and port */ -union Socket { - - using Address = ip4::Addr; +class Socket { +public: + using Address = net::Addr; using port_t = uint16_t; /** @@ -37,7 +37,12 @@ union Socket { * Intialize an empty socket <0.0.0.0:0> */ constexpr Socket() noexcept - : sock_(0, 0) {} + : Socket{0} + {} + + explicit constexpr Socket(const port_t port) noexcept + : addr_{}, port_{port} + {} /** * Constructor @@ -50,15 +55,20 @@ union Socket { * @param port * The port associated with the process */ - constexpr Socket(const Address address, const port_t port) noexcept - : sock_(address, port) {} - - Socket(const Socket& other) noexcept - : data_{other.data_} {} + Socket(Address address, const port_t port) noexcept + : addr_{std::move(address)}, + port_{port} + {} - Socket& operator=(const Socket& other) noexcept + Socket(const Socket& other) noexcept = default; + Socket(Socket&& other) noexcept + : addr_{std::move(other.addr_)}, port_{other.port_} + {} + Socket& operator=(const Socket& other) noexcept = default; + Socket& operator=(Socket&& other) noexcept { - data_ = other.data_; + addr_ = std::move(other.addr_); + port_ = other.port_; return *this; } @@ -67,8 +77,8 @@ union Socket { * * @return The socket's network address */ - constexpr Address address() const noexcept - { return sock_.address; } + const Address& address() const noexcept + { return addr_; } /** * Get the socket's port value @@ -76,7 +86,7 @@ union Socket { * @return The socket's port value */ constexpr port_t port() const noexcept - { return sock_.port; } + { return port_; } /** * Get a string representation of this class @@ -84,15 +94,15 @@ union Socket { * @return A string representation of this class */ std::string to_string() const - { return address().str() + ":" + std::to_string(port()); } + { return address().to_string() + ":" + std::to_string(port()); } /** * Check if this socket is empty <0.0.0.0:0> * * @return true if this socket is empty, false otherwise */ - constexpr bool is_empty() const noexcept - { return (address() == 0) and (port() == 0); } + bool is_empty() const noexcept + { return (addr_.v6() == ip6::Addr::link_unspecified) and (port() == 0); } /** * Operator to check for equality relationship @@ -102,9 +112,9 @@ union Socket { * * @return true if the specified socket is equal, false otherwise */ - constexpr bool operator==(const Socket& other) const noexcept + bool operator==(const Socket& other) const noexcept { - return data_ == other.data_; + return addr_ == other.addr_; } /** @@ -115,7 +125,7 @@ union Socket { * * @return true if the specified socket is not equal, false otherwise */ - constexpr bool operator!=(const Socket& other) const noexcept + bool operator!=(const Socket& other) const noexcept { return not (*this == other); } /** @@ -127,7 +137,7 @@ union Socket { * @return true if this socket is less-than the specified socket, * false otherwise */ - constexpr bool operator<(const Socket& other) const noexcept + bool operator<(const Socket& other) const noexcept { return (address() < other.address()) or ((address() == other.address()) and (port() < other.port())); @@ -142,21 +152,18 @@ union Socket { * @return true if this socket is greater-than the specified socket, * false otherwise */ - constexpr bool operator>(const Socket& other) const noexcept + bool operator>(const Socket& other) const noexcept { return not (*this < other); } private: - struct Sock { - constexpr Sock(Address a, port_t p) : address(a), port(p), padding(0) {} - Address address; - port_t port; - uint16_t padding; - } sock_; - uint64_t data_; + Address addr_; + port_t port_; }; //< class Socket -static_assert(sizeof(Socket) == 8, "Socket not 8 byte"); +static_assert(std::is_move_constructible_v); +static_assert(std::is_move_assignable_v); +//static_assert(sizeof(Socket) == 18, "Socket not 18 byte"); /** * @brief A pair of Sockets @@ -202,7 +209,7 @@ namespace std { struct hash { public: size_t operator () (const net::Socket& key) const noexcept { - const auto h1 = std::hash{}(key.address()); + const auto h1 = std::hash{}(key.address().v6()); const auto h2 = std::hash{}(key.port()); return h1 ^ h2; } diff --git a/api/net/tcp/common.hpp b/api/net/tcp/common.hpp index 07233011af..4aab679603 100644 --- a/api/net/tcp/common.hpp +++ b/api/net/tcp/common.hpp @@ -19,7 +19,7 @@ #ifndef NET_TCP_COMMON_HPP #define NET_TCP_COMMON_HPP -#include +#include #include #include #include @@ -48,7 +48,7 @@ namespace net { static const std::chrono::seconds default_msl {30}; static const std::chrono::milliseconds default_dack_timeout {40}; - using Address = ip4::Addr; + using Address = net::Addr; /** A port */ using port_t = uint16_t; diff --git a/api/net/tcp/listener.hpp b/api/net/tcp/listener.hpp index 7cf4a6693e..189a4e3c8a 100644 --- a/api/net/tcp/listener.hpp +++ b/api/net/tcp/listener.hpp @@ -42,7 +42,8 @@ class Listener { public: - Listener(TCP& host, Socket local, ConnectCallback cb = nullptr); + Listener(TCP& host, Socket local, ConnectCallback cb = nullptr, + const bool ipv6_only = false); Listener& on_accept(AcceptCallback cb) { @@ -96,6 +97,7 @@ class Listener { AcceptCallback on_accept_; ConnectCallback on_connect_; CloseCallback _on_close_; + const bool ipv6_only_; bool default_on_accept(Socket); diff --git a/api/net/tcp/packet.hpp b/api/net/tcp/packet.hpp index 37f867af78..3dabd74dbf 100644 --- a/api/net/tcp/packet.hpp +++ b/api/net/tcp/packet.hpp @@ -134,13 +134,13 @@ class Packet : public PacketIP4 { Packet& set_source(const Socket& src) { - set_ip_src(src.address()); // PacketIP4::set_src + set_ip_src(src.address().v4()); // PacketIP4::set_src set_src_port(src.port()); return *this; } Packet& set_destination(const Socket& dest) { - set_ip_dst(dest.address()); // PacketIP4::set_dst + set_ip_dst(dest.address().v4()); // PacketIP4::set_dst set_dst_port(dest.port()); return *this; } diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index cde1512e69..02617134c5 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -49,11 +49,13 @@ namespace net { using Packet_reroute_func = delegate; + using Port_utils = std::map; + friend class tcp::Connection; friend class tcp::Listener; private: - using Listeners = std::map>; + using Listeners = std::map>; using Connections = std::map; public: @@ -77,8 +79,8 @@ namespace net { * * @return A TCP Listener */ - tcp::Listener& listen(const tcp::port_t port, ConnectCallback cb = nullptr) - { return listen({0, port}, std::move(cb)); } + tcp::Listener& listen(const tcp::port_t port, ConnectCallback cb = nullptr, + const bool ipv6_only = false); /** * @brief Bind to a socket to start listening for new connections @@ -477,7 +479,7 @@ namespace net { Listeners listeners_; Connections connections_; - IP4::Port_utils& ports_; + Port_utils& ports_; downstream _network_layer_out; @@ -614,7 +616,7 @@ namespace net { * * @return True if valid source, False otherwise. */ - bool is_valid_source(const tcp::Address addr) const noexcept; + bool is_valid_source(const tcp::Address& addr) const noexcept; /** * @brief Try to find the listener bound to socket. @@ -624,13 +626,7 @@ namespace net { * * @return A listener iterator */ - Listeners::iterator find_listener(const Socket socket) - { - Listeners::iterator it = listeners_.find(socket); - if(it == listeners_.end() and socket.address() != 0) - it = listeners_.find({0, socket.port()}); - return it; - } + Listeners::iterator find_listener(const Socket& socket); /** * @brief Try to find the listener bound to socket. @@ -640,13 +636,7 @@ namespace net { * * @return A listener const iterator */ - Listeners::const_iterator cfind_listener(const Socket socket) const - { - Listeners::const_iterator it = listeners_.find(socket); - if(it == listeners_.cend() and socket.address() != 0) - it = listeners_.find({0, socket.port()}); - return it; - } + Listeners::const_iterator cfind_listener(const Socket& socket) const; /** * @brief Adds a connection. diff --git a/api/util/syslog_facility.hpp b/api/util/syslog_facility.hpp index 0c18e338c9..ddf463aeb1 100644 --- a/api/util/syslog_facility.hpp +++ b/api/util/syslog_facility.hpp @@ -175,7 +175,7 @@ class Syslog_udp : public Syslog_facility { ~Syslog_udp(); private: - net::UDP::addr_t ip_{0}; + net::UDP::addr_t ip_{}; net::UDP::port_t port_{0}; net::UDPSocket* sock_ = nullptr; @@ -187,7 +187,7 @@ class Syslog_print : public Syslog_facility { public: void syslog(const std::string& log_message) override; void settings(const net::UDP::addr_t, const net::UDP::port_t) override {} - net::UDP::addr_t ip() const noexcept override { return 0; } + net::UDP::addr_t ip() const noexcept override { return net::ip6::Addr::addr_any; } net::UDP::port_t port() const noexcept override { return 0; } void open_socket() override {} void close_socket() override {} diff --git a/lib/mender/src/client.cpp b/lib/mender/src/client.cpp index e71bef0213..2f87df9c79 100644 --- a/lib/mender/src/client.cpp +++ b/lib/mender/src/client.cpp @@ -28,7 +28,7 @@ namespace mender { : am_{std::forward(man)}, device_{std::forward(dev)}, server_(server), - cached_{0, port}, + cached_{net::Addr{}, port}, httpclient_{std::make_unique(tcp)}, state_(&state::Init::instance()), context_({this, &Client::run_state}), diff --git a/lib/microLB/micro_lb/autoconf.cpp b/lib/microLB/micro_lb/autoconf.cpp index 8b1dc88a39..3c9c498b00 100644 --- a/lib/microLB/micro_lb/autoconf.cpp +++ b/lib/microLB/micro_lb/autoconf.cpp @@ -61,7 +61,7 @@ namespace microLB assert(port >= 0 && port < 65536); // try to construct socket from string net::Socket socket{ - {addr[0].GetString()}, (uint16_t) port + net::ip4::Addr{addr[0].GetString()}, (uint16_t) port }; balancer->nodes.add_node(netout, socket, balancer->get_pool_signal()); } diff --git a/src/net/dhcp/dh4client.cpp b/src/net/dhcp/dh4client.cpp index d987bd2f4d..2124932e21 100644 --- a/src/net/dhcp/dh4client.cpp +++ b/src/net/dhcp/dh4client.cpp @@ -142,7 +142,7 @@ namespace net { /// broadcast our DHCP plea as 0.0.0.0:67 socket->bcast(IP4::ADDR_ANY, DHCP_SERVER_PORT, buffer, sizeof(buffer)); socket->on_read( - [this] (IP4::addr addr, UDP::port_t port, + [this] (net::Addr addr, UDP::port_t port, const char* data, size_t len) { if (port == DHCP_SERVER_PORT) @@ -292,7 +292,7 @@ namespace net { assert(this->socket); // set our onRead function to point to a hopeful DHCP ACK! socket->on_read( - [this] (IP4::addr addr, UDP::port_t port, + [this] (net::Addr addr, UDP::port_t port, const char* data, size_t len) { if (port == DHCP_SERVER_PORT) diff --git a/src/net/dhcp/dhcpd.cpp b/src/net/dhcp/dhcpd.cpp index 3958f6611b..09718605d3 100644 --- a/src/net/dhcp/dhcpd.cpp +++ b/src/net/dhcp/dhcpd.cpp @@ -94,7 +94,7 @@ void DHCPD::update_pool(IP4::addr ip, Status new_status) { } void DHCPD::listen() { - socket_.on_read([&] (IP4::addr, UDP::port_t port, + socket_.on_read([&] (net::Addr, UDP::port_t port, const char* data, size_t len) { if (port == DHCP_CLIENT_PORT) { diff --git a/src/net/dns/client.cpp b/src/net/dns/client.cpp index 597e8a68cd..6a580add59 100644 --- a/src/net/dns/client.cpp +++ b/src/net/dns/client.cpp @@ -59,7 +59,7 @@ namespace net if(it != cache_.end()) { Error err; - func(it->second.address, err); + func(it->second.address.v4(), err); return; } } @@ -98,7 +98,7 @@ namespace net flush_timer_.stop(); } - void DNSClient::receive_response(IP4::addr, UDP::port_t, const char* data, size_t) + void DNSClient::receive_response(Address, UDP::port_t, const char* data, size_t) { const auto& reply = *(DNS::header*) data; // match the transactions id on the reply with the ones in our map diff --git a/src/net/ip4/ip4.cpp b/src/net/ip4/ip4.cpp index e6e94e7b2f..ae0ed5d8ef 100644 --- a/src/net/ip4/ip4.cpp +++ b/src/net/ip4/ip4.cpp @@ -31,6 +31,7 @@ namespace net { + const ip4::Addr ip4::Addr::addr_any{0}; const IP4::addr IP4::ADDR_ANY(0); const IP4::addr IP4::ADDR_BCAST(0xff,0xff,0xff,0xff); @@ -327,7 +328,7 @@ namespace net { if (UNLIKELY(not path_mtu_discovery_ or dest.address() == IP4::ADDR_ANY or (new_pmtu > 0 and new_pmtu < minimum_MTU()))) return; - if (UNLIKELY(dest.address().is_multicast())) { + if (UNLIKELY(dest.address().v4().is_multicast())) { // TODO RFC4821 p. 12 } diff --git a/src/net/ip4/udp.cpp b/src/net/ip4/udp.cpp index 695431eba6..fc74c50c46 100644 --- a/src/net/ip4/udp.cpp +++ b/src/net/ip4/udp.cpp @@ -135,7 +135,7 @@ namespace net { return it.first->second; } - UDPSocket& UDP::bind(const ip4::Addr addr) + UDPSocket& UDP::bind(const addr_t addr) { if(UNLIKELY( not stack_.is_valid_source(addr) )) throw UDP_error{"Cannot bind to address: " + addr.to_string()}; @@ -290,8 +290,8 @@ namespace net { // Initialize UDP packet p2->init(l_port, d_port); - p2->set_ip_src(l_addr); - p2->set_ip_dst(d_addr); + p2->set_ip_src(l_addr.v4()); + p2->set_ip_dst(d_addr.v4()); p2->set_data_length(total); // fill buffer (at payload position) @@ -315,7 +315,7 @@ namespace net { } } - ip4::Addr UDP::local_ip() const + UDP::addr_t UDP::local_ip() const { return stack_.ip_addr(); } UDPSocket& UDP::bind(port_t port) diff --git a/src/net/ip4/udp_socket.cpp b/src/net/ip4/udp_socket.cpp index 707486c4d6..72d42f3eee 100644 --- a/src/net/ip4/udp_socket.cpp +++ b/src/net/ip4/udp_socket.cpp @@ -33,8 +33,8 @@ namespace net uint16_t length) { p->init(this->local_port(), port); - p->set_ip_src(srcIP); - p->set_ip_dst(destIP); + p->set_ip_src(srcIP.v4()); + p->set_ip_dst(destIP.v4()); p->set_data_length(length); assert(p->data_length() == length); diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index 82fb98fcf4..43883aed03 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -36,6 +36,7 @@ namespace net const ip6::Addr ip6::Addr::node_mDNSv6(0xFF01, 0, 0, 0, 0, 0, 0, 0xFB); const ip6::Addr ip6::Addr::link_unspecified(0, 0, 0, 0, 0, 0, 0, 0); + const ip6::Addr ip6::Addr::addr_any{ip6::Addr::link_unspecified}; const ip6::Addr ip6::Addr::link_all_nodes(0xFF02, 0, 0, 0, 0, 0, 0, 1); const ip6::Addr ip6::Addr::link_all_routers(0xFF02, 0, 0, 0, 0, 0, 0, 2); diff --git a/src/net/nat/napt.cpp b/src/net/nat/napt.cpp index 5fada464b4..a88668b45b 100644 --- a/src/net/nat/napt.cpp +++ b/src/net/nat/napt.cpp @@ -48,7 +48,7 @@ inline void update_dnat(Conntrack& ct, Conntrack::Entry_ptr entry, Socket socket inline void update_dnat(Conntrack& ct, Conntrack::Entry_ptr entry, ip4::Addr addr) { - if(entry->second.src.address() != addr) { + if(entry->second.src.address().v4() != addr) { ct.update_entry(entry->proto, entry->second, { {addr, entry->second.src.port()}, // change return addr but keep port entry->second.dst @@ -78,7 +78,7 @@ inline void update_snat(Conntrack& ct, Conntrack::Entry_ptr entry, Socket socket inline void update_snat(Conntrack& ct, Conntrack::Entry_ptr entry, ip4::Addr addr) { - if(entry->second.dst.address() != addr) { + if(entry->second.dst.address().v4() != addr) { ct.update_entry(entry->proto, entry->second, { entry->second.src, {addr, entry->second.dst.port()} // change dst address but keep port @@ -146,7 +146,7 @@ void NAPT::masquerade(IP4::IP_packet& pkt, Stack& inet, Conntrack::Entry_ptr ent NATDBG(" MASQ: %s => %s\n", entry->to_string().c_str(), entry->second.dst.to_string().c_str()); // static source nat - icmp_snat(pkt, entry->second.dst.address()); + icmp_snat(pkt, entry->second.dst.address().v4()); break; } @@ -179,7 +179,7 @@ void NAPT::demasquerade(IP4::IP_packet& pkt, const Stack&, Conntrack::Entry_ptr break; case Protocol::ICMPv4: - icmp_dnat(pkt, entry->first.src.address()); + icmp_dnat(pkt, entry->first.src.address().v4()); break; default: @@ -338,7 +338,7 @@ void NAPT::dnat(IP4::IP_packet& p, Conntrack::Entry_ptr entry) { NATDBG(" Found DNAT target: %s => %s\n", entry->to_string().c_str(), entry->first.src.address().to_string().c_str()); - icmp_dnat(p, entry->first.src.address()); + icmp_dnat(p, entry->first.src.address().v4()); } return; } @@ -472,7 +472,7 @@ void NAPT::snat(IP4::IP_packet& p, Conntrack::Entry_ptr entry) { NATDBG(" Found SNAT target: %s => %s\n", entry->to_string().c_str(), entry->first.dst.address().to_string().c_str()); - icmp_snat(p, entry->first.dst.address()); + icmp_snat(p, entry->first.dst.address().v4()); } return; } diff --git a/src/net/nat/nat.cpp b/src/net/nat/nat.cpp index 6f2cc82856..24b5fc72fd 100644 --- a/src/net/nat/nat.cpp +++ b/src/net/nat/nat.cpp @@ -213,7 +213,7 @@ void udp_snat(PacketIP4& p, Socket new_sock) // Recalc checksum recalc_udp_sock(pkt, old_sock, new_sock); // Set the value - pkt.set_ip_src(new_sock.address()); + pkt.set_ip_src(new_sock.address().v4()); pkt.set_src_port(new_sock.port()); } @@ -248,7 +248,7 @@ void udp_dnat(PacketIP4& p, Socket new_sock) recalc_udp_sock(pkt, old_sock, new_sock); // change destination - pkt.set_ip_dst(new_sock.address()); + pkt.set_ip_dst(new_sock.address().v4()); pkt.set_dst_port(new_sock.port()); } @@ -296,8 +296,8 @@ inline void recalc_ip_checksum(PacketIP4& pkt, ip4::Addr old_addr, ip4::Addr new inline void recalc_tcp_sock(tcp::Packet& pkt, Socket osock, Socket nsock) { - auto old_addr = osock.address(); - auto new_addr = nsock.address(); + auto old_addr = osock.address().v4(); + auto new_addr = nsock.address().v4(); // recalc IP checksum recalc_ip_checksum(pkt, old_addr, new_addr); @@ -336,8 +336,8 @@ inline void recalc_tcp_port(tcp::Packet& pkt, uint16_t old_port, uint16_t new_po inline void recalc_udp_sock(PacketUDP& pkt, Socket osock, Socket nsock) { - auto old_addr = osock.address(); - auto new_addr = nsock.address(); + auto old_addr = osock.address().v4(); + auto new_addr = nsock.address().v4(); // recalc IP checksum recalc_ip_checksum(pkt, old_addr, new_addr); diff --git a/src/net/tcp/listener.cpp b/src/net/tcp/listener.cpp index bb34da7c95..d87a30d1ab 100644 --- a/src/net/tcp/listener.cpp +++ b/src/net/tcp/listener.cpp @@ -22,11 +22,12 @@ using namespace net; using namespace tcp; -Listener::Listener(TCP& host, Socket local, ConnectCallback cb) +Listener::Listener(TCP& host, Socket local, ConnectCallback cb, const bool ipv6_only) : host_(host), local_(local), syn_queue_(), on_accept_({this, &Listener::default_on_accept}), on_connect_{std::move(cb)}, - _on_close_({host_, &TCP::close_listener}) + _on_close_({host_, &TCP::close_listener}), + ipv6_only_{ipv6_only} { } @@ -152,5 +153,6 @@ std::string Listener::to_string() const { // add syn queue for(auto& conn : syn_queue_) str += "\n\t" + conn->to_string(); + return str; } diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index 5005e1cd4a..43bb165655 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -82,17 +82,6 @@ void TCP::smp_process_writeq(size_t packets) SMP::signal(this->cpu_id); } -/* - Note: There is different approaches to how to handle listeners & connections. - Need to discuss and decide for the best one. - - Best solution(?): - Preallocate a pool with listening connections. - When threshold is reach, remove/add new ones, similar to TCP window. - - Current solution: - Simple. -*/ Listener& TCP::listen(Socket socket, ConnectCallback cb) { bind(socket); @@ -104,7 +93,26 @@ Listener& TCP::listen(Socket socket, ConnectCallback cb) return *listener; } +Listener& TCP::listen(const tcp::port_t port, ConnectCallback cb, const bool ipv6_only) +{ + Socket socket{ip6::Addr::addr_any, port}; + bind(socket); + + auto ptr = std::make_shared(*this, socket, std::move(cb), ipv6_only); + auto& listener = listeners_.emplace(socket, ptr).first->second; + + if(not ipv6_only) + { + Socket ip4_sock{ip4::Addr::addr_any, port}; + bind(ip4_sock); + Ensures(listeners_.emplace(ip4_sock, ptr).second && "Could not insert IPv4 listener"); + } + + return *listener; +} + bool TCP::close(Socket socket) { + //TODO: check ip6 any addr auto it = listeners_.find(socket); if(it != listeners_.end()) { @@ -177,7 +185,7 @@ void TCP::receive(net::Packet_ptr packet_ptr) { } const auto dest = packet->destination(); - debug2(" TCP Packet received - Source: %s, Destination: %s \n", + debug(" TCP Packet received - Source: %s, Destination: %s \n", packet->source().to_string().c_str(), dest.to_string().c_str()); // Validate checksum @@ -216,7 +224,7 @@ void TCP::receive(net::Packet_ptr packet_ptr) { // Listener found => Create Listener if (listener_it != listeners_.end()) { auto& listener = listener_it->second; - debug(" Listener found: %s\n", listener->to_string().c_str()); + debug2(" Listener found: %s\n", listener->to_string().c_str()); listener->segment_arrived(std::move(packet)); debug2(" Listener done with packet\n"); return; @@ -234,8 +242,8 @@ string TCP::to_string() const { // Write all connections in a cute list. std::string str = "LISTENERS:\nLocal\tQueued\n"; for(auto& listen_it : listeners_) { - auto& l = listen_it.second; - str += l->local().to_string() + "\t" + std::to_string(l->syn_queue_size()) + "\n"; + auto& l = listen_it; + str += l.first.to_string() + "\t" + std::to_string(l.second->syn_queue_size()) + "\n"; } str += "\nCONNECTIONS:\nProto\tRecv\tSend\tIn\tOut\tLocal\t\t\tRemote\t\t\tState\n"; @@ -380,8 +388,8 @@ bool TCP::is_bound(const Socket socket) const { auto it = ports_.find(socket.address()); - if(it == ports_.cend() and socket.address() != 0) - it = ports_.find({0}); + if(it == ports_.cend() and not socket.address().is_any()) + it = ports_.find(socket.address().any_addr()); if(it != ports_.cend()) return it->second.is_bound(socket.port()); @@ -502,8 +510,35 @@ tcp::Address TCP::address() const noexcept IP4& TCP::network() const { return inet_.ip_obj(); } -bool TCP::is_valid_source(const tcp::Address addr) const noexcept -{ return addr == 0 or inet_.is_valid_source(addr); } +bool TCP::is_valid_source(const tcp::Address& addr) const noexcept +{ return addr.is_any() or inet_.is_valid_source(addr); } void TCP::kick() { process_writeq(inet_.transmit_queue_available()); } + +TCP::Listeners::iterator TCP::find_listener(const Socket& socket) +{ + Listeners::iterator it = listeners_.find(socket); + if(it != listeners_.end()) + return it; + + if(not socket.address().is_any()) + it = listeners_.find({socket.address().any_addr(), socket.port()}); + + return it; +} + +TCP::Listeners::const_iterator TCP::cfind_listener(const Socket& socket) const +{ + Listeners::const_iterator it = listeners_.find(socket); + if(it != listeners_.cend()) + return it; + + if(not socket.address().is_any()) + it = listeners_.find({socket.address().any_addr(), socket.port()}); + + return it; +} + + + diff --git a/src/plugins/unik.cpp b/src/plugins/unik.cpp index e5e86de995..5695c7bce0 100644 --- a/src/plugins/unik.cpp +++ b/src/plugins/unik.cpp @@ -30,7 +30,7 @@ unik::Client::Registered_event unik::Client::on_registered_{nullptr}; void unik::Client::register_instance(net::Inet& inet, const net::UDP::port_t port) { INFO("Unik client", "Initializing Unik registration service"); - INFO("Unik client","Listening for UDP hearbeat on %s:%i", inet.ip_addr().str().c_str(), port); + INFO("Unik client","Listening for UDP hearbeat on %s:%i", inet.ip_addr().to_string().c_str(), port); INFO("Unik client","IP is attached to interface %s ", inet.link_addr().str().c_str()); // Set up an UDP port for receiving UniK heartbeat @@ -46,7 +46,7 @@ void unik::Client::register_instance(net::Inet& inet, const net::UDP::port_t por return; std::string strdata(data, len); - INFO("Unik client","received UDP data from %s:%i: %s ", addr.str().c_str(), port, strdata.c_str()); + INFO("Unik client","received UDP data from %s:%i: %s ", addr.to_string().c_str(), port, strdata.c_str()); auto dotloc = strdata.find(":"); diff --git a/src/posix/tcp_fd.cpp b/src/posix/tcp_fd.cpp index ca9cf1778c..a2cf6115bb 100644 --- a/src/posix/tcp_fd.cpp +++ b/src/posix/tcp_fd.cpp @@ -388,7 +388,7 @@ long TCP_FD_Listen::accept(struct sockaddr *__restrict__ addr, socklen_t *__rest auto* sin = (sockaddr_in*) addr; sin->sin_family = AF_INET; sin->sin_port = conn->remote().port(); - sin->sin_addr.s_addr = conn->remote().address().whole; + sin->sin_addr.s_addr = conn->remote().address().v4().whole; *addr_len = sizeof(sockaddr_in); } // return socket diff --git a/src/posix/udp_fd.cpp b/src/posix/udp_fd.cpp index 9e96b30dfa..dda5882432 100644 --- a/src/posix/udp_fd.cpp +++ b/src/posix/udp_fd.cpp @@ -43,7 +43,7 @@ void UDP_FD::recv_to_buffer(net::UDPSocket::addr_t addr, // copy data into to-be Message buffer auto buff = net::tcp::buffer_t(new std::vector (buf, buf + len)); // emplace the message in buffer - buffer_.emplace_back(htonl(addr.whole), htons(port), std::move(buff)); + buffer_.emplace_back(htonl(addr.v4().whole), htons(port), std::move(buff)); } } @@ -177,7 +177,7 @@ ssize_t UDP_FD::sendto(const void* message, size_t len, int, // Sending bool written = false; - this->sock->sendto(ntohl(dest.sin_addr.s_addr), ntohs(dest.sin_port), message, len, + this->sock->sendto(net::ip4::Addr{ntohl(dest.sin_addr.s_addr)}, ntohs(dest.sin_port), message, len, [&written]() { written = true; }); while(!written) @@ -230,7 +230,7 @@ ssize_t UDP_FD::recvfrom(void *__restrict__ buffer, size_t len, int flags, auto& sender = *((sockaddr_in*)address); sender.sin_family = AF_INET; sender.sin_port = htons(port); - sender.sin_addr.s_addr = htonl(addr.whole); + sender.sin_addr.s_addr = htonl(addr.v4().whole); *address_len = sizeof(struct sockaddr_in); } done = true; From 00a9caca930b30b23ab0df01c6e7e4509470c46c Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 4 Jun 2018 15:02:35 +0200 Subject: [PATCH 385/723] pci: Read BARs into the correct indices --- api/hw/pci_device.hpp | 26 ++++++++++++-------------- src/drivers/e1000.cpp | 2 +- src/drivers/virtioblk.cpp | 4 +--- src/drivers/vmxnet3.cpp | 4 ++-- src/hw/msi.cpp | 11 ++++++----- src/hw/pci_device.cpp | 24 ++++++++++-------------- src/kernel/os.cpp | 2 -- 7 files changed, 32 insertions(+), 41 deletions(-) diff --git a/api/hw/pci_device.hpp b/api/hw/pci_device.hpp index 8abef14440..18a427745c 100644 --- a/api/hw/pci_device.hpp +++ b/api/hw/pci_device.hpp @@ -113,12 +113,8 @@ namespace PCI { static inline const char* vendor_str(uint16_t code); struct Resource { - int type; - uint32_t start; - uint32_t len; - Resource* next; - Resource(int t, const uint32_t Start, const uint32_t Len) - : type(t), start{Start}, len{Len}, next(nullptr) {} + uintptr_t start; + size_t len; }; static const uint8_t RES_IO = 0; @@ -207,8 +203,10 @@ struct msix_t; */ void probe_resources() noexcept; - /** The base address of the (first) I/O resource */ - uint32_t iobase() const noexcept; + /** The base address of the I/O resource */ + uint32_t iobase() const { + return m_resources.at(this->m_iobase).start; + } typedef uint32_t pcicap_t; void parse_capabilities(); @@ -242,14 +240,15 @@ struct msix_t; void init_msix(); // resource handling - uintptr_t get_bar(uint8_t id) const noexcept + const PCI::Resource& get_bar(uint8_t id) const { - return resources.at(id).start; + return m_resources.at(id); } bool validate_bar(uint8_t id) const noexcept { - return id < resources.size(); + return id < m_resources.size(); } + void* allocate_bar(uint8_t id, int pages); // @brief The 2-part ID retrieved from the device union vendor_product_t { @@ -284,10 +283,9 @@ struct msix_t; vendor_product_t device_id_; class_revision_t devtype_; - // Device Resources - typedef PCI::Resource Resource; //! @brief List of PCI BARs - std::vector resources; + int m_iobase = -1; + std::array m_resources; pcicap_t caps[PCI_CAP_ID_MAX+1]; diff --git a/src/drivers/e1000.cpp b/src/drivers/e1000.cpp index bb60a0ff72..5684750a7c 100644 --- a/src/drivers/e1000.cpp +++ b/src/drivers/e1000.cpp @@ -137,7 +137,7 @@ e1000::e1000(hw::PCI_Device& d, uint16_t mtu) : } // shared-memory & I/O address - this->shm_base = d.get_bar(0); + this->shm_base = d.get_bar(0).start; this->use_mmio = this->shm_base > 0xFFFF; if (this->use_mmio == false) { this->io_base = d.iobase(); diff --git a/src/drivers/virtioblk.cpp b/src/drivers/virtioblk.cpp index c4da500ffb..8f55c3b020 100644 --- a/src/drivers/virtioblk.cpp +++ b/src/drivers/virtioblk.cpp @@ -1,5 +1,3 @@ -#define DEBUG -#define DEBUG2 #include "virtioblk.hpp" #include @@ -37,7 +35,7 @@ void null_deleter(uint8_t*) {}; VirtioBlk::VirtioBlk(hw::PCI_Device& d) : Virtio(d), hw::Block_device(), req(device_name() + ".req0", queue_size(0), 0, iobase()), inflight(0) { - INFO("VirtioBlk", "Block_devicer initializing"); + INFO("VirtioBlk", "Initializing"); { auto& reqs = Statman::get().create( Stat::UINT32, device_name() + ".requests"); diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index e3ef157be6..c5cb72a02a 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -152,9 +152,9 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : } // dma areas - this->iobase = d.get_bar(PCI_BAR_VD); + this->iobase = d.get_bar(PCI_BAR_VD).start; assert(this->iobase); - this->ptbase = d.get_bar(PCI_BAR_PT); + this->ptbase = d.get_bar(PCI_BAR_PT).start; assert(this->ptbase); // verify and select version diff --git a/src/hw/msi.cpp b/src/hw/msi.cpp index 21a294fcbc..8cdc8c2428 100644 --- a/src/hw/msi.cpp +++ b/src/hw/msi.cpp @@ -104,12 +104,13 @@ namespace hw printf("PCI: Invalid BAR: %u\n", bar); return 0; } - auto baroff = dev.get_bar(bar); - assert(baroff != 0); + auto pcibar = dev.get_bar(bar); + assert(pcibar.start != 0); PRINT("[MSI-X] offset %p -> bir %u => %p res: %p\n", - (void*) offset, bar, (void*) baroff, (void*) (capbar_off + baroff)); - return capbar_off + baroff; + (void*) offset, bar, (void*) pcibar.start, + (void*) (pcibar.start + capbar_off)); + return pcibar.start + capbar_off; } msix_t::msix_t(PCI_Device& device, uint32_t cap) @@ -153,7 +154,7 @@ namespace hw for (size_t i = 0; i < this->vectors(); i++) { mask_entry(i); } - PRINT("[MSI-X] Enabled with %u vectors\n", this->vectors()); + PRINT("[MSI-X] Enabled with %zu vectors\n", this->vectors()); // unmask vectors func &= ~MSIX_FUNC_MASK; diff --git a/src/hw/pci_device.cpp b/src/hw/pci_device.cpp index 7eee9073ac..632ba6f70f 100644 --- a/src/hw/pci_device.cpp +++ b/src/hw/pci_device.cpp @@ -67,14 +67,6 @@ namespace hw { return size; } - uint32_t PCI_Device::iobase() const noexcept { - - for (auto& res : resources) { - if (res.type == PCI::RES_IO) return res.start; - } - assert(0 && "No I/O resource present on device"); - } - void PCI_Device::probe_resources() noexcept { //Find resources on this PCI device (scan the BAR's) @@ -100,19 +92,17 @@ namespace hw { // Resource type IO unmasked_val = value & PCI::BASE_ADDRESS_IO_MASK; pci__size = pci_size(len, PCI::BASE_ADDRESS_IO_MASK & 0xFFFF); - - resources.emplace_back(PCI::RES_IO, unmasked_val, pci__size); + this->m_iobase = bar; } else { // Resource type Mem unmasked_val = value & PCI::BASE_ADDRESS_MEM_MASK; pci__size = pci_size(len, PCI::BASE_ADDRESS_MEM_MASK); - - resources.emplace_back(PCI::RES_MEM, unmasked_val, pci__size); } + this->m_resources.at(bar) = {unmasked_val, pci__size}; - INFO2("| |- BAR %s @ 0x%x, size %i ", - value & 1 ? "I/O" : "Mem", unmasked_val, pci__size); + INFO2("| |- BAR%d %s @ 0x%x, size %i ", bar, + (value & 1 ? "I/O" : "Mem"), unmasked_val, pci__size); } } @@ -139,6 +129,12 @@ namespace hw { bridge_subclasses[devtype_.subclass < SS_BR ? devtype_.subclass : SS_BR-1], devtype_.subclass); break; + case PCI::classcode::STORAGE: + INFO2("+--[ %s, %s (0x%x) ]", + PCI::classcode_str(devtype_.classcode), + PCI::vendor_str(vendor_id()), + product_id()); + break; case PCI::classcode::NIC: INFO2("+--[ %s, %s, %s (0x%x) ]", PCI::classcode_str(devtype_.classcode), diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 389e5c1407..ea5cb60ddd 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -145,7 +145,6 @@ void OS::post_start() // Run plugin constructors __run_ctors(&__plugin_ctors_start, &__plugin_ctors_end); - // Run plugins PROFILE("Plugins init"); for (auto plugin : plugins) { @@ -184,7 +183,6 @@ bool os_default_stdout = false; void OS::print(const char* str, const size_t len) { - if (UNLIKELY(! __libc_initialized)) { OS::default_stdout(str, len); return; From 55097a8cbe4c8e5f8be2261fe4302001bd1649b4 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 4 Jun 2018 15:02:52 +0200 Subject: [PATCH 386/723] vmrunner: Add NVMe blk support --- vmrunner/vm.schema.json | 2 +- vmrunner/vmrunner.py | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/vmrunner/vm.schema.json b/vmrunner/vm.schema.json index 1bde0fc7d3..faefbf8ee6 100644 --- a/vmrunner/vm.schema.json +++ b/vmrunner/vm.schema.json @@ -54,7 +54,7 @@ "type" : "object", "properties" : { "file" : { "type" : "string" }, - "type" : { "enum" : ["ide", "virtio"] }, + "type" : { "enum" : ["ide", "virtio", "virtio-scsi", "nvme"] }, "format" : { "enum" : ["raw", "qcow2", "vdi"] }, "media" : { "enum" : ["disk"] }, "name" : { "type" : "string" } diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 231776337d..897b2eb6e6 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -331,11 +331,25 @@ def name(self): def image_name(self): return self._image_name - def drive_arg(self, filename, drive_type = "virtio", drive_format = "raw", media_type = "disk"): - return ["-drive","file=" + filename - + ",format=" + drive_format - + ",if=" + drive_type - + ",media=" + media_type] + def drive_arg(self, filename, device = "virtio", drive_format = "raw", media_type = "disk"): + names = {"virtio" : "virtio-blk", + "virtio-scsi" : "virtio-scsi", + "ide" : "piix3-ide", + "nvme" : "nvme"} + + if device == "ide": + # most likely a problem relating to bus, or wrong .drive + return ["-drive","file=" + filename + + ",format=" + drive_format + + ",if=" + device + + ",media=" + media_type] + else: + if device in names: + device = names[device] + + return ["-drive", "file=" + filename + ",format=" + drive_format + + ",if=none" + ",media=" + media_type + ",id=drv0", + "-device", device + ",drive=drv0,serial=foo"] # -initrd "file1 arg=foo,file2" # This syntax is only available with multiboot. From b1a341703cbc6adb215b11b81cfce08280e5d4d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 5 Jun 2018 11:02:49 +0200 Subject: [PATCH 387/723] net: Don't reset ip6 config, fix some debug --- api/net/inet.hpp | 8 ++++---- api/net/ip6/addr.hpp | 17 +++++++++++------ src/net/dhcp/dh4client.cpp | 4 ++-- src/net/inet.cpp | 6 +++--- src/net/ip6/ndp.cpp | 8 +++++--- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/api/net/inet.hpp b/api/net/inet.hpp index 803f73945b..856efc983f 100644 --- a/api/net/inet.hpp +++ b/api/net/inet.hpp @@ -88,7 +88,7 @@ namespace net { IP4::addr broadcast_addr() const { return ip4_addr_ | ( ~ netmask_); } - IP6::addr ip6_addr() const + const ip6::Addr& ip6_addr() const noexcept { return ip6_addr_; } uint8_t netmask6() const @@ -295,9 +295,9 @@ namespace net { this->ip4_addr_ = IP4::ADDR_ANY; this->gateway_ = IP4::ADDR_ANY; this->netmask_ = IP4::ADDR_ANY; - this->ip6_addr_ = IP6::ADDR_ANY; - this->ip6_gateway_ = IP6::ADDR_ANY; - this->ip6_prefix_ = 0; + //this->ip6_addr_ = IP6::ADDR_ANY; + //this->ip6_gateway_ = IP6::ADDR_ANY; + //this->ip6_prefix_ = 0; } // register a callback for receiving signal on free packet-buffers diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index 1e93717a7e..7894b0c720 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -58,10 +58,11 @@ struct Addr { i32[2] = htonl(c); i32[3] = htonl(d); } - Addr(const Addr& a) - : i32{a.i32} {} + Addr(const Addr& a) noexcept + : i64{a.i64} {} - Addr(Addr&& a) = default; + Addr(Addr&& a) noexcept + : i64{std::move(a.i64)} {} // returns this IPv6 Address as a string std::string str() const { @@ -169,13 +170,17 @@ struct Addr { /** * Assignment operator */ - Addr& operator=(const Addr& other) + Addr& operator=(const Addr& other) noexcept { - i32 = other.i32; + i64 = other.i64; return *this; } - Addr& operator=(Addr&& other) = default; + Addr& operator=(Addr&& other) noexcept + { + i64 = std::move(other.i64); + return *this; + } /** * Operator to check for equality diff --git a/src/net/dhcp/dh4client.cpp b/src/net/dhcp/dh4client.cpp index 2124932e21..52fefbabf5 100644 --- a/src/net/dhcp/dh4client.cpp +++ b/src/net/dhcp/dh4client.cpp @@ -150,7 +150,7 @@ namespace net { // we have got a DHCP Offer (void) addr; PRINT("Received possible DHCP OFFER from %s\n", - addr.str().c_str()); + addr.to_string().c_str()); this->offer(data, len); } }); @@ -300,7 +300,7 @@ namespace net { (void) addr; // we have hopefully got a DHCP Ack PRINT("\tReceived DHCP ACK from %s:%d\n", - addr.str().c_str(), DHCP_SERVER_PORT); + addr.to_string().c_str(), DHCP_SERVER_PORT); this->acknowledge(data, len); } }); diff --git a/src/net/inet.cpp b/src/net/inet.cpp index 1dfc93879e..0d76211bcf 100644 --- a/src/net/inet.cpp +++ b/src/net/inet.cpp @@ -222,11 +222,11 @@ void Inet::network_config6(IP6::addr addr6, IP6::addr gateway6) { - this->ip6_addr_ = addr6; + this->ip6_addr_ = std::move(addr6); this->ip6_prefix_ = prefix6; - this->ip6_gateway_ = gateway6; + this->ip6_gateway_ = std::move(gateway6); INFO("Inet6", "Network configured (%s)", nic_.mac().to_string().c_str()); - INFO2("IP6: \t\t%s", ip6_addr_.str().c_str()); + INFO2("IP6: \t\t%s", ip6_addr_.to_string().c_str()); INFO2("Prefix: \t%d", ip6_prefix_); INFO2("Gateway: \t%s", ip6_gateway_.str().c_str()); diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index 812d99bc8e..bc14e00b12 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -73,9 +73,10 @@ namespace net // Add checksum res.set_checksum(); - PRINT("NDP: Neighbor Adv Response dst: %s size: %i payload size: %i," - " checksum: 0x%x\n", res.ip().size(), res.payload().size(), - res.ip().ip_dst().str().c_str(), res.compute_checksum()); + PRINT("NDP: Neighbor Adv Response dst: %s\n size: %i\n payload size: %i\n," + " checksum: 0x%x\n", + res.ip().ip_dst().str().c_str(), res.ip().size(), res.payload().size(), + res.compute_checksum()); auto dest = res.ip().ip_dst(); transmit(res.release(), dest); @@ -196,6 +197,7 @@ namespace net bool is_dest_multicast = req.ip().ip_dst().is_multicast(); if (target != inet_.ip6_addr()) { + PRINT("NDP: not for us. target=%s us=%s\n", target.to_string().c_str(), inet_.ip6_addr().to_string().c_str()); /* Not for us. Should we forward? */ return; } From c15756eb02143934b423019d08ca5d2b93a96cc8 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 5 Jun 2018 16:00:41 +0200 Subject: [PATCH 388/723] drivers: add room for vlan tags in packet_len --- src/drivers/e1000.hpp | 3 ++- src/drivers/virtionet.hpp | 3 ++- src/drivers/vmxnet3.hpp | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/drivers/e1000.hpp b/src/drivers/e1000.hpp index 0402b363b8..5817170825 100644 --- a/src/drivers/e1000.hpp +++ b/src/drivers/e1000.hpp @@ -18,6 +18,7 @@ #include #include #include +#include // vlan header size #include #include @@ -47,7 +48,7 @@ class e1000 : public net::Link_layer } uint16_t packet_len() const noexcept { - return sizeof(net::ethernet::Header) + MTU(); + return sizeof(net::ethernet::VLAN_header) + MTU(); } net::downstream create_physical_downstream() override diff --git a/src/drivers/virtionet.hpp b/src/drivers/virtionet.hpp index 5bec9a0838..ae7612e16c 100644 --- a/src/drivers/virtionet.hpp +++ b/src/drivers/virtionet.hpp @@ -39,6 +39,7 @@ #include #include #include +#include // vlan header size #include #include #include @@ -133,7 +134,7 @@ class VirtioNet : Virtio, public net::Link_layer { { return 1500; } uint16_t packet_len() const noexcept { - return sizeof(net::ethernet::Header) + MTU(); + return sizeof(net::ethernet::VLAN_header) + MTU(); } net::Packet_ptr create_packet(int) override; diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index 9c4c6688ce..1aea496efe 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include struct vmxnet3_dma; struct vmxnet3_rx_desc; @@ -51,7 +52,7 @@ class vmxnet3 : public net::Link_layer } uint16_t packet_len() const noexcept { - return sizeof(net::ethernet::Header) + MTU(); + return sizeof(net::ethernet::VLAN_header) + MTU(); } net::downstream create_physical_downstream() override From 5f513749ed46ed7c33bb1d7ce1fbba439aaa7fbb Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 5 Jun 2018 16:47:43 +0200 Subject: [PATCH 389/723] e1000: Correctly calculate buffer for higher MTUs --- src/drivers/e1000.cpp | 5 +++-- src/drivers/e1000.hpp | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/drivers/e1000.cpp b/src/drivers/e1000.cpp index 5684750a7c..a5372cabdb 100644 --- a/src/drivers/e1000.cpp +++ b/src/drivers/e1000.cpp @@ -56,9 +56,10 @@ static inline uint16_t report_size_for_mtu(uint16_t mtu) static inline uint16_t buffer_size_for_mtu(const uint16_t mtu) { const uint16_t header = sizeof(net::Packet) + e1000::DRIVER_OFFSET; - if (mtu <= 2048 - header) return 2048; + const uint16_t total = header + sizeof(net::ethernet::VLAN_header) + mtu; + if (total <= 2048) return 2048; assert(mtu <= 9000 && "Buffers larger than 9000 are not supported"); - return report_size_for_mtu(mtu) + header; + return total; } #define NUM_PACKET_BUFFERS (NUM_TX_DESC + NUM_RX_DESC + NUM_TX_QUEUE + 8) diff --git a/src/drivers/e1000.hpp b/src/drivers/e1000.hpp index 5817170825..660598c171 100644 --- a/src/drivers/e1000.hpp +++ b/src/drivers/e1000.hpp @@ -17,8 +17,7 @@ #include #include -#include -#include // vlan header size +#include #include #include From 90e49b37632de4c5363844c30435aa9f6604ab42 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 5 Jun 2018 16:48:01 +0200 Subject: [PATCH 390/723] vmxnet3: Correctly calculate buffer for higher MTUs --- src/drivers/vmxnet3.cpp | 7 ++++--- src/drivers/vmxnet3.hpp | 1 - 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index c5cb72a02a..2323b38a79 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -110,9 +110,10 @@ inline void mmio_write32(uintptr_t location, uint32_t value) static inline uint16_t buffer_size_for_mtu(const uint16_t mtu) { const uint16_t header = sizeof(net::Packet) + vmxnet3::DRIVER_OFFSET; - if (mtu <= 2048 - header) return 2048; - assert(header + mtu <= 16384 && "Buffers larger than 16k are not supported"); - return mtu + header; + const uint16_t total = header + sizeof(net::ethernet::VLAN_header) + mtu; + if (total <= 2048) return 2048; + assert(total <= 16384 && "Buffers larger than 16k are not supported"); + return total; } vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index 1aea496efe..59fdad8dbe 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -17,7 +17,6 @@ #include #include -#include #include #include struct vmxnet3_dma; From f7fad0daccf2f42fa9e15dcf48be7dd55ebcb89b Mon Sep 17 00:00:00 2001 From: niks3089 Date: Tue, 5 Jun 2018 21:07:37 +0530 Subject: [PATCH 391/723] ndp: fix warnings --- api/net/ip6/ndp.hpp | 6 +++--- api/net/ip6/packet_icmp6.hpp | 5 +++-- src/net/ip6/ip6.cpp | 2 +- src/net/ip6/ndp.cpp | 3 +-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/api/net/ip6/ndp.hpp b/api/net/ip6/ndp.hpp index 456acebcd6..9da5c36707 100644 --- a/api/net/ip6/ndp.hpp +++ b/api/net/ip6/ndp.hpp @@ -96,7 +96,7 @@ namespace net { void cache(IP6::addr ip, uint8_t *ll_addr, NeighbourStates state, uint32_t flags); /** Lookup for cache entry */ - bool lookup(IP6::addr ip, uint8_t *ll_addr); + bool lookup(IP6::addr ip); /** Flush the NDP cache */ void flush_cache() @@ -182,10 +182,10 @@ namespace net { } private: - uint32_t flags_; MAC::Addr mac_; - RTC::timestamp_t timestamp_; NeighbourStates state_; + RTC::timestamp_t timestamp_; + uint32_t flags_; }; //< struct Cache_entry struct Queue_entry { diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index 4b0e8b7ad3..fc97a36e3e 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -82,8 +82,9 @@ namespace icmp6 { } public: - NdpOptions() : header_{nullptr}, opt_array{}, nd_opts_ri{nullptr}, - nd_opts_ri_end{nullptr}, user_opts{nullptr}, user_opts_end{nullptr} {} + NdpOptions() : header_{nullptr}, nd_opts_ri{nullptr}, + nd_opts_ri_end{nullptr}, user_opts{nullptr}, + user_opts_end{nullptr}, opt_array{} {} void parse(uint8_t *opt, uint16_t opts_len); struct nd_options_header *get_header(uint8_t &opt) diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index 4a0e537952..540e231eea 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -113,7 +113,7 @@ namespace net return next_proto; } - void IP6::receive(Packet_ptr pckt, const bool link_bcast) + void IP6::receive(Packet_ptr pckt, const bool /*link_bcast */) { // Cast to IP6 Packet auto packet = static_unique_ptr_cast(std::move(pckt)); diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index 812d99bc8e..6391a344ef 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -162,7 +162,6 @@ namespace net bool any_src = req.ip().ip_src() == IP6::ADDR_ANY; IP6::addr target = req.ndp().neighbour_sol().get_target(); uint8_t *lladdr, *nonce_opt; - bool found; uint64_t nonce = 0; PRINT("ICMPv6 NDP Neighbor solicitation request\n"); @@ -270,7 +269,7 @@ namespace net } } - bool Ndp::lookup(IP6::addr ip, uint8_t *ll_addr) + bool Ndp::lookup(IP6::addr ip) { auto entry = neighbour_cache_.find(ip); if (entry != neighbour_cache_.end()) { From 8e937b53080fd7da34aecf6e41df3d1ecb8bdc82 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 6 Jun 2018 10:46:28 +0200 Subject: [PATCH 392/723] uplink: Don't list all certificates, just total --- lib/uplink/ws_uplink.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index 69b0321c24..e75caa3335 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -56,13 +56,12 @@ static SSL_CTX* init_ssl_context(const std::string& certs_path, bool verify) auto ents = disk.fs().ls(certs_path); int files = 0; - MYINFO("Scanning files..."); for(auto& ent : ents) { if(not ent.is_file()) continue; - INFO2("%s", ent.name().c_str()); files++; } + INFO2("%d certificates", files); Expects(files > 0 && "No files found on disk"); From 1e618b754b73beac5143d9fc47b55afdee639b00 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 6 Jun 2018 10:48:08 +0200 Subject: [PATCH 393/723] cmake: Generate memdisk.asm to build folder --- cmake/post.service.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index cf6cdd649b..b909f70d88 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -361,8 +361,8 @@ function(add_memdisk DISK) REALPATH BASE_DIR "${CMAKE_SOURCE_DIR}") add_custom_command( OUTPUT memdisk.o - COMMAND python ${INSTALL_LOC}/memdisk/memdisk.py --file ${INSTALL_LOC}/memdisk/memdisk.asm ${DISK_RELPATH} - COMMAND nasm -f ${CMAKE_ASM_NASM_OBJECT_FORMAT} ${INSTALL_LOC}/memdisk/memdisk.asm -o memdisk.o + COMMAND python ${INSTALL_LOC}/memdisk/memdisk.py --file memdisk.asm ${DISK_RELPATH} + COMMAND nasm -f ${CMAKE_ASM_NASM_OBJECT_FORMAT} memdisk.asm -o memdisk.o DEPENDS ${DISK_RELPATH} fake_news ) add_library(memdisk STATIC memdisk.o) @@ -452,7 +452,7 @@ target_link_libraries(service libos libplatform libarch - + ${LIBR_CMAKE_NAMES} libos libbotan From 0bf6c557352e026ba86a6b38695741ab47aaf69a Mon Sep 17 00:00:00 2001 From: taiyeba Date: Thu, 7 Jun 2018 09:37:28 +0200 Subject: [PATCH 394/723] net/icmp6: silencing the output from ipv6 command --- test/net/integration/icmp6/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net/integration/icmp6/setup.sh b/test/net/integration/icmp6/setup.sh index 4e6f22a543..797956d30c 100755 --- a/test/net/integration/icmp6/setup.sh +++ b/test/net/integration/icmp6/setup.sh @@ -1,2 +1,2 @@ #!/bin/bash -sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 +sudo sysctl -w net.ipv6.conf.all.disable_ipv6=0 &> /dev/null From ad8719f00cc29e26655b800e0992e7781dd9cb4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 7 Jun 2018 10:44:39 +0200 Subject: [PATCH 395/723] tcp: Added Packet_view to replace the current tcp::Packet --- api/net/tcp/common.hpp | 8 +++++ api/net/tcp/packet4_view.hpp | 44 +++++++++++++++++++++++ api/net/tcp/packet6_view.hpp | 44 +++++++++++++++++++++++ api/net/tcp/packet_view.hpp | 68 ++++++++++++++++++++++++++++++++++++ 4 files changed, 164 insertions(+) create mode 100644 api/net/tcp/packet4_view.hpp create mode 100644 api/net/tcp/packet6_view.hpp create mode 100644 api/net/tcp/packet_view.hpp diff --git a/api/net/tcp/common.hpp b/api/net/tcp/common.hpp index 4aab679603..e1a6a80d00 100644 --- a/api/net/tcp/common.hpp +++ b/api/net/tcp/common.hpp @@ -90,6 +90,14 @@ namespace net { return net::checksum(sum, buffer, length); } + template + uint16_t calculate_checksum6(const View6& packet) + { + // Alf fix me ;D + (void) packet; + return 0; + } + } // < namespace tcp } // < namespace net diff --git a/api/net/tcp/packet4_view.hpp b/api/net/tcp/packet4_view.hpp new file mode 100644 index 0000000000..25fdee601f --- /dev/null +++ b/api/net/tcp/packet4_view.hpp @@ -0,0 +1,44 @@ + +#pragma once + +#include +#include + +namespace net::tcp { + +class Packet4_view : public Packet_view { +public: + Packet4_view(std::unique_ptr ptr) + : Packet_view(std::move(ptr)) + { + Expects(packet().is_ipv4()); + header = reinterpret_cast(packet().ip_data().data()); + } + + uint16_t compute_tcp_checksum() noexcept override + { return calculate_checksum6(*this); } + +private: + PacketIP4& packet() noexcept + { return static_cast(*pkt); } + + const PacketIP4& packet() const noexcept + { return static_cast(*pkt); } + + void set_ip_src(const net::Addr& addr) noexcept override + { packet().set_ip_src(addr.v4()); } + + void set_ip_dst(const net::Addr& addr) noexcept override + { packet().set_ip_dst(addr.v4()); } + + net::Addr ip_src() const noexcept override + { return packet().ip_src(); } + + net::Addr ip_dst() const noexcept override + { return packet().ip_dst(); } + + uint16_t ip_data_length() const noexcept override + { return packet().ip_data_length(); } +}; + +} diff --git a/api/net/tcp/packet6_view.hpp b/api/net/tcp/packet6_view.hpp new file mode 100644 index 0000000000..f643588b2f --- /dev/null +++ b/api/net/tcp/packet6_view.hpp @@ -0,0 +1,44 @@ + +#pragma once + +#include +#include + +namespace net::tcp { + +class Packet6_view : public Packet_view { +public: + Packet6_view(std::unique_ptr ptr) + : Packet_view(std::move(ptr)) + { + Expects(packet().is_ipv6()); + header = reinterpret_cast(packet().ip_data().data()); + } + + uint16_t compute_tcp_checksum() noexcept override + { return calculate_checksum6(*this); } + +private: + PacketIP6& packet() noexcept + { return static_cast(*pkt); } + + const PacketIP6& packet() const noexcept + { return static_cast(*pkt); } + + void set_ip_src(const net::Addr& addr) noexcept override + { packet().set_ip_src(addr.v6()); } + + void set_ip_dst(const net::Addr& addr) noexcept override + { packet().set_ip_dst(addr.v6()); } + + net::Addr ip_src() const noexcept override + { return packet().ip_src(); } + + net::Addr ip_dst() const noexcept override + { return packet().ip_dst(); } + + uint16_t ip_data_length() const noexcept override + { return packet().ip_data_length(); } +}; + +} diff --git a/api/net/tcp/packet_view.hpp b/api/net/tcp/packet_view.hpp new file mode 100644 index 0000000000..c3fa7b78bc --- /dev/null +++ b/api/net/tcp/packet_view.hpp @@ -0,0 +1,68 @@ + +#pragma once + +#include +namespace net::tcp { + +class Packet_view { +public: + Packet_view(net::Packet_ptr ptr) + : pkt{std::move(ptr)} + { + Expects(pkt != nullptr); + } + + Header& tcp_header() noexcept + { return *header; } + + const Header& tcp_header() const noexcept + { return *header; } + + port_t src_port() const noexcept + { return ntohs(tcp_header().source_port); } + + port_t dst_port() const noexcept + { return ntohs(tcp_header().destination_port); } + + Socket source() const noexcept + { return Socket{ip_src(), src_port()}; } + + Socket destination() const noexcept + { return Socket{ip_dst(), dst_port()}; } + + + // Get the raw tcp offset, in quadruples + int offset() const + { return tcp_header().offset_flags.offset_reserved >> 4; } + + // The actual TCP header size (including options). + auto tcp_header_length() const + { return offset() * 4; } + + // Length of data in packet when header has been accounted for + uint16_t tcp_data_length() const + { return ip_data_length() - tcp_header_length(); } + + + + net::Packet_ptr release() + { + Expects(pkt != nullptr && "Packet ptr is already null"); + return std::move(pkt); + } + + virtual uint16_t compute_tcp_checksum() noexcept = 0; + +protected: + net::Packet_ptr pkt; + Header* header = nullptr; + + virtual void set_ip_src(const net::Addr& addr) noexcept = 0; + virtual void set_ip_dst(const net::Addr& addr) noexcept = 0; + virtual net::Addr ip_src() const noexcept = 0; + virtual net::Addr ip_dst() const noexcept = 0; + // TODO: see if we can get rid of this virtual call + virtual uint16_t ip_data_length() const noexcept = 0; +}; + +} From ded3678a78e54b202d362f9ab738ff17e917b8b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 7 Jun 2018 10:45:37 +0200 Subject: [PATCH 396/723] tcp: Now receives IP6 packets --- api/net/tcp/tcp.hpp | 10 ++++++ src/net/inet.cpp | 4 +++ src/net/ip6/ip6.cpp | 2 +- src/net/tcp/tcp.cpp | 76 ++++++++++++++++++++++++++++++--------------- 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index 02617134c5..779da354a0 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -24,6 +24,7 @@ #include "headers.hpp" #include "listener.hpp" #include "packet.hpp" +#include "packet_view.hpp" #include // connections, listeners #include // writeq @@ -180,6 +181,15 @@ namespace net { */ void receive(net::Packet_ptr); + /** + * @brief Receive a Packet from the network layer (IP6) + * + * @param[in] A IP6 packet + */ + void receive6(net::Packet_ptr); + + void receive(tcp::Packet_view&); + /** * @brief Sets a delegate to the network output. * diff --git a/src/net/inet.cpp b/src/net/inet.cpp index 0d76211bcf..e64aea72ab 100644 --- a/src/net/inet.cpp +++ b/src/net/inet.cpp @@ -47,6 +47,7 @@ Inet::Inet(hw::Nic& nic) auto icmp6_bottom(upstream{icmp6_, &ICMPv6::receive}); auto udp4_bottom(upstream{udp_, &UDP::receive}); auto tcp_bottom(upstream{tcp_, &TCP::receive}); + auto tcp6_bottom(upstream{tcp_, &TCP::receive6}); /** Upstream wiring */ // Packets available @@ -73,6 +74,9 @@ Inet::Inet(hw::Nic& nic) // IP4 -> TCP ip4_.set_tcp_handler(tcp_bottom); + // IP6 -> TCP + ip6_.set_tcp_handler(tcp6_bottom); + /** Downstream delegates */ auto link_top(nic_.create_link_downstream()); auto arp_top(IP4::downstream_arp{arp_, &Arp::transmit}); diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index 72c5157d43..7fe71c83ba 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -222,7 +222,7 @@ namespace net //udp_handler_(std::move(packet)); break; case Protocol::TCP: - //tcp_handler_(std::move(packet)); + tcp_handler_(std::move(packet)); break; default: // Send ICMP error of type Destination Unreachable and code PROTOCOL diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index 43bb165655..00ba9398d0 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -15,15 +15,20 @@ // See the License for the specific language governing permissions and // limitations under the License. -//#undef NO_DEBUG -#define DEBUG -#define DEBUG2 +//#define TCP_DEBUG 1 +#ifdef TCP_DEBUG +#define PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define PRINT(fmt, ...) /* fmt */ +#endif #include #include #include // checksum #include #include // nanos_now (get_ts_value) +#include +#include using namespace std; using namespace net; @@ -170,50 +175,70 @@ void TCP::insert_connection(Connection_ptr conn) std::forward_as_tuple(conn)); } -void TCP::receive(net::Packet_ptr packet_ptr) { +void TCP::receive(net::Packet_ptr ptr) +{ + auto ip4 = static_unique_ptr_cast(std::move(ptr)); + Packet4_view pkt{std::move(ip4)}; + + PRINT(" Recv TCP4 packet %s => %s\n", + pkt.source().to_string().c_str(), pkt.destination().to_string().c_str()); + + receive(pkt); +} + +void TCP::receive6(net::Packet_ptr ptr) +{ + auto ip6 = static_unique_ptr_cast(std::move(ptr)); + Packet6_view pkt{std::move(ip6)}; + + PRINT(" Recv TCP6 packet %s => %s\n", + pkt.source().to_string().c_str(), pkt.destination().to_string().c_str()); + + receive(pkt); +} + +void TCP::receive(Packet_view& packet) +{ // Stat increment packets received (*packets_rx_)++; assert(get_cpuid() == SMP::cpu_id()); - // Translate into a TCP::Packet. This will be used inside the TCP-scope. - auto packet = static_unique_ptr_cast(std::move(packet_ptr)); + //auto packet = static_unique_ptr_cast(std::move(packet_ptr)); // validate some unlikely but invalid packet properties - if (UNLIKELY(packet->src_port() == 0)) { - drop(*packet); + if (UNLIKELY(packet.src_port() == 0)) { + drop(*static_unique_ptr_cast(packet.release())); return; } - const auto dest = packet->destination(); - debug(" TCP Packet received - Source: %s, Destination: %s \n", - packet->source().to_string().c_str(), dest.to_string().c_str()); + const auto dest = packet.destination(); // Validate checksum - if (UNLIKELY(packet->compute_tcp_checksum() != 0)) { - debug(" TCP Packet Checksum %#x != %#x\n", - packet->compute_tcp_checksum(), 0x0); - drop(*packet); + if (UNLIKELY(packet.compute_tcp_checksum() != 0)) { + PRINT(" TCP Packet Checksum %#x != %#x\n", + packet.compute_tcp_checksum(), 0x0); + drop(*static_unique_ptr_cast(packet.release())); return; } // Stat increment bytes received - (*bytes_rx_) += packet->tcp_data_length(); + (*bytes_rx_) += packet.tcp_data_length(); // Redirect packet to custom function if (packet_rerouter) { - packet_rerouter(std::move(packet)); + packet_rerouter(static_unique_ptr_cast(packet.release())); return; } - const Connection::Tuple tuple { dest, packet->source() }; + const Connection::Tuple tuple { dest, packet.source() }; // Try to find the receiver auto conn_it = connections_.find(tuple); // Connection found if (conn_it != connections_.end()) { - debug(" Connection found: %s \n", conn_it->second->to_string().c_str()); - conn_it->second->segment_arrived(std::move(packet)); + PRINT(" Connection found: %s \n", conn_it->second->to_string().c_str()); + conn_it->second->segment_arrived(static_unique_ptr_cast(packet.release())); return; } @@ -224,16 +249,17 @@ void TCP::receive(net::Packet_ptr packet_ptr) { // Listener found => Create Listener if (listener_it != listeners_.end()) { auto& listener = listener_it->second; - debug2(" Listener found: %s\n", listener->to_string().c_str()); - listener->segment_arrived(std::move(packet)); - debug2(" Listener done with packet\n"); + PRINT(" Listener found: %s\n", listener->to_string().c_str()); + listener->segment_arrived(static_unique_ptr_cast(packet.release())); + PRINT(" Listener done with packet\n"); return; } + auto pkt = static_unique_ptr_cast(packet.release()); // Send a reset - send_reset(*packet); + send_reset(*pkt); - drop(*packet); + drop(*pkt); } // Show all connections for TCP as a string. From 1418b60742bc2b56291580857bf02dc7cf8220b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 7 Jun 2018 10:48:24 +0200 Subject: [PATCH 397/723] vmrunner: silence some qemu boot stuff --- vmrunner/vmrunner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 231776337d..1505b0f926 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -371,6 +371,7 @@ def net_arg(self, backend, device, if_name = "net0", mac = None, bridge = None): # Add mac-address if specified if mac: device += ",mac=" + mac + device += ",romfile=" # remove some qemu boot info (experimental) return ["-device", device, "-netdev", netdev] From 909b2b32bcc5f706f1d2329e8a000efb49f61dba Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Thu, 7 Jun 2018 11:46:07 +0200 Subject: [PATCH 398/723] Test dhcpd: Add flushing of arp cache for addresses used in test --- test/README.md | 2 +- test/net/integration/dhcpd/service.cpp | 2 +- test/net/integration/dhcpd/setup.sh | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) create mode 100755 test/net/integration/dhcpd/setup.sh diff --git a/test/README.md b/test/README.md index 382199cdf3..645fb4ef9f 100644 --- a/test/README.md +++ b/test/README.md @@ -11,7 +11,7 @@ The following IP addresses are used by the services in the test folder. Make sur - bufstore: No IP - configure: 10.0.0.60, 10.0.0.61, 10.0.0.62 - dhclient: Time_sensitive, leave for now -- dhcpd: 10.0.0.1, 10.0.0.10, 10.0.0.11, 10.0.0.12 - Is 10.0.0.1 a problem? +- dhcpd: 10.0.0.9, 10.0.0.10, 10.0.0.11, 10.0.0.12 - dhcpd_dhclient_linux: leave for now - dns: 10.0.0.48 - gateway: 10.0.1.1, 10.0.1.10, 10.0.2.1, 10.0.2.10 diff --git a/test/net/integration/dhcpd/service.cpp b/test/net/integration/dhcpd/service.cpp index 59db449902..6d2f778fe5 100644 --- a/test/net/integration/dhcpd/service.cpp +++ b/test/net/integration/dhcpd/service.cpp @@ -30,7 +30,7 @@ void Service::start(const std::string&) // Server auto& inet = Inet::ifconfig<0>( - { 10,0,0,1 }, // IP + { 10,0,0,9 }, // IP { 255,255,255,0 }, // Netmask { 10,0,0,1 }, // Gateway { 8,8,8,8 }); // DNS diff --git a/test/net/integration/dhcpd/setup.sh b/test/net/integration/dhcpd/setup.sh new file mode 100755 index 0000000000..83924c1d3c --- /dev/null +++ b/test/net/integration/dhcpd/setup.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +sudo arp -d 10.0.0.10 2>&1 > /dev/null +sudo arp -d 10.0.0.11 2>&1 > /dev/null +sudo arp -d 10.0.0.12 2>&1 > /dev/null From 9900973fad5954a3c622534c79637a511d2c2967 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 7 Jun 2018 13:36:10 +0200 Subject: [PATCH 399/723] tcp: Replace recvside with Packet_view, also store ts opt --- api/net/tcp/connection.hpp | 35 +++-- api/net/tcp/connection_states.hpp | 22 +-- api/net/tcp/listener.hpp | 4 +- api/net/tcp/packet4_view.hpp | 2 +- api/net/tcp/packet6_view.hpp | 2 +- api/net/tcp/packet_view.hpp | 80 +++++++++- api/net/tcp/tcp.hpp | 4 +- src/net/tcp/connection.cpp | 33 ++-- src/net/tcp/connection_states.cpp | 251 +++++++++++++++--------------- src/net/tcp/listener.cpp | 21 ++- src/net/tcp/tcp.cpp | 20 ++- 11 files changed, 273 insertions(+), 201 deletions(-) diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index bca1e2db58..d6cbc80a6f 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -21,6 +21,7 @@ #include "common.hpp" #include "packet.hpp" +#include "packet_view.hpp" #include "read_request.hpp" #include "rttm.hpp" #include "tcp_errors.hpp" @@ -460,7 +461,7 @@ class Connection { virtual void abort(Connection&); /** Handle a Packet [SEGMENT ARRIVES] */ - virtual Result handle(Connection&, Packet_ptr in) = 0; + virtual Result handle(Connection&, Packet_view& in) = 0; /** The current state represented as a string [STATUS] */ virtual std::string to_string() const = 0; @@ -485,13 +486,13 @@ class Connection { Helper functions TODO: Clean up names. */ - virtual bool check_seq(Connection&, const Packet&); + virtual bool check_seq(Connection&, Packet_view&); - virtual void unallowed_syn_reset_connection(Connection&, const Packet&); + virtual void unallowed_syn_reset_connection(Connection&, const Packet_view&); - virtual bool check_ack(Connection&, const Packet&); + virtual bool check_ack(Connection&, const Packet_view&); - virtual void process_fin(Connection&, const Packet&); + virtual void process_fin(Connection&, const Packet_view&); virtual void send_reset(Connection&); @@ -817,7 +818,7 @@ class Connection { /* Drop a packet. Used for debug/callback. */ - void drop(const Packet& packet, Drop_reason reason = Drop_reason::NA); + void drop(const Packet_view& packet, Drop_reason reason = Drop_reason::NA); // RFC 3042 void limited_tx(); @@ -861,22 +862,22 @@ class Connection { /* Receive a TCP Packet. */ - void segment_arrived(Packet_ptr); + void segment_arrived(Packet_view&); /* Acknowledge a packet - TCB update, Congestion control handling, RTT calculation and RT handling. */ - bool handle_ack(const Packet&); + bool handle_ack(const Packet_view&); /** * @brief Receive data from an incoming packet containing data. * * @param[in] in TCP Packet containing payload */ - void recv_data(const Packet& in); + void recv_data(const Packet_view& in); - void recv_out_of_order(const Packet& in); + void recv_out_of_order(const Packet_view& in); /** * @brief Acknowledge incoming data. This is done by: @@ -893,7 +894,7 @@ class Connection { * * @return True if window update, False otherwise. */ - bool is_win_update(const Packet& in, const uint32_t win) const + bool is_win_update(const Packet_view& in, const uint32_t win) const { return cb.SND.WND != win and (cb.SND.WL1 < in.seq() or (cb.SND.WL1 == in.seq() and cb.SND.WL2 <= in.ack())); @@ -906,7 +907,7 @@ class Connection { * * @return True if duplicate acknowledge, False otherwise. */ - bool is_dup_ack(const Packet& in, const uint32_t win) const + bool is_dup_ack(const Packet_view& in, const uint32_t win) const { return in.ack() == cb.SND.UNA and flight_size() > 0 @@ -920,21 +921,21 @@ class Connection { * * @param[in] Incoming TCP segment (duplicate ACK) */ - void on_dup_ack(const Packet&); + void on_dup_ack(const Packet_view&); /** * @brief Handle segment according to congestion control (New Reno) * * @param[in] Incoming TCP segment */ - void congestion_control(const Packet&); + void congestion_control(const Packet_view&); /** * @brief Handle segment according to fast recovery (New Reno) * * @param[in] Incoming TCP segment */ - void fast_recovery(const Packet&); + void fast_recovery(const Packet_view&); /** * @brief Determines ability to send ONE segment, not caring about the usable window. @@ -1060,7 +1061,7 @@ class Connection { * * @param[in] An incomming TCP packet */ - void take_rtt_measure(const Packet&); + void take_rtt_measure(const Packet_view&); /* Start retransmission timer. @@ -1149,7 +1150,7 @@ class Connection { /* Parse and apply options. */ - void parse_options(const Packet&); + void parse_options(const Packet_view&); /* Add an option. diff --git a/api/net/tcp/connection_states.hpp b/api/net/tcp/connection_states.hpp index 79ee1ce23b..869ae69de5 100644 --- a/api/net/tcp/connection_states.hpp +++ b/api/net/tcp/connection_states.hpp @@ -53,7 +53,7 @@ class Connection::Closed : public State { => SynSent */ - Result handle(Connection&, Packet_ptr in) override; + Result handle(Connection&, Packet_view& in) override; bool is_closed() const override { return true; @@ -87,7 +87,7 @@ class Connection::Listen : public State { => SynReceived. */ - virtual Result handle(Connection&, Packet_ptr in) override; + virtual Result handle(Connection&, Packet_view& in) override; inline virtual std::string to_string() const override { return "LISTENING"; @@ -116,7 +116,7 @@ class Connection::SynSent : public State { => Established. */ - virtual Result handle(Connection&, Packet_ptr in) override; + virtual Result handle(Connection&, Packet_view& in) override; inline virtual std::string to_string() const override { return "SYN-SENT"; @@ -151,7 +151,7 @@ class Connection::SynReceived : public State { => Established. */ - virtual Result handle(Connection&, Packet_ptr in) override; + virtual Result handle(Connection&, Packet_view& in) override; inline virtual std::string to_string() const override { return "SYN-RCV"; @@ -177,7 +177,7 @@ class Connection::Established : public State { virtual void abort(Connection&) override; - virtual Result handle(Connection&, Packet_ptr in) override; + virtual Result handle(Connection&, Packet_view& in) override; inline virtual std::string to_string() const override { return "ESTABLISHED"; @@ -218,7 +218,7 @@ class Connection::FinWait1 : public State { => FinWait2. */ - virtual Result handle(Connection&, Packet_ptr in) override; + virtual Result handle(Connection&, Packet_view& in) override; inline virtual std::string to_string() const override { return "FIN-WAIT-1"; @@ -252,7 +252,7 @@ class Connection::FinWait2 : public State { /* */ - virtual Result handle(Connection&, Packet_ptr in) override; + virtual Result handle(Connection&, Packet_view& in) override; inline virtual std::string to_string() const override { return "FIN-WAIT-2"; @@ -292,7 +292,7 @@ class Connection::CloseWait : public State { => LastAck */ - virtual Result handle(Connection&, Packet_ptr in) override; + virtual Result handle(Connection&, Packet_view& in) override; inline virtual std::string to_string() const override { return "CLOSE-WAIT"; @@ -320,7 +320,7 @@ class Connection::Closing : public State { => TimeWait (Guess this isnt needed, just start a Close-timer) */ - virtual Result handle(Connection&, Packet_ptr in) override; + virtual Result handle(Connection&, Packet_view& in) override; inline virtual std::string to_string() const override { return "CLOSING"; @@ -350,7 +350,7 @@ class Connection::LastAck : public State { => Closed (Tell TCP to remove this connection) */ - virtual Result handle(Connection&, Packet_ptr in) override; + virtual Result handle(Connection&, Packet_view& in) override; inline virtual std::string to_string() const override { return "LAST-ACK"; @@ -376,7 +376,7 @@ class Connection::TimeWait : public State { /* */ - virtual Result handle(Connection&, Packet_ptr in) override; + virtual Result handle(Connection&, Packet_view& in) override; std::string to_string() const override { return "TIME-WAIT"; diff --git a/api/net/tcp/listener.hpp b/api/net/tcp/listener.hpp index 189a4e3c8a..230bab90b6 100644 --- a/api/net/tcp/listener.hpp +++ b/api/net/tcp/listener.hpp @@ -23,7 +23,7 @@ #include "common.hpp" #include "connection.hpp" -#include "packet.hpp" +#include "packet_view.hpp" #include @@ -101,7 +101,7 @@ class Listener { bool default_on_accept(Socket); - void segment_arrived(Packet_ptr); + void segment_arrived(Packet_view&); void remove(Connection_ptr); diff --git a/api/net/tcp/packet4_view.hpp b/api/net/tcp/packet4_view.hpp index 25fdee601f..5bd6a987e3 100644 --- a/api/net/tcp/packet4_view.hpp +++ b/api/net/tcp/packet4_view.hpp @@ -12,7 +12,7 @@ class Packet4_view : public Packet_view { : Packet_view(std::move(ptr)) { Expects(packet().is_ipv4()); - header = reinterpret_cast(packet().ip_data().data()); + set_header(packet().ip_data().data()); } uint16_t compute_tcp_checksum() noexcept override diff --git a/api/net/tcp/packet6_view.hpp b/api/net/tcp/packet6_view.hpp index f643588b2f..c45154ab2d 100644 --- a/api/net/tcp/packet6_view.hpp +++ b/api/net/tcp/packet6_view.hpp @@ -12,7 +12,7 @@ class Packet6_view : public Packet_view { : Packet_view(std::move(ptr)) { Expects(packet().is_ipv6()); - header = reinterpret_cast(packet().ip_data().data()); + set_header(packet().ip_data().data()); } uint16_t compute_tcp_checksum() noexcept override diff --git a/api/net/tcp/packet_view.hpp b/api/net/tcp/packet_view.hpp index c3fa7b78bc..4a795e7854 100644 --- a/api/net/tcp/packet_view.hpp +++ b/api/net/tcp/packet_view.hpp @@ -1,7 +1,9 @@ #pragma once -#include +#include "common.hpp" +#include "options.hpp" + namespace net::tcp { class Packet_view { @@ -18,6 +20,8 @@ class Packet_view { const Header& tcp_header() const noexcept { return *header; } + // Ports etc // + port_t src_port() const noexcept { return ntohs(tcp_header().source_port); } @@ -30,32 +34,86 @@ class Packet_view { Socket destination() const noexcept { return Socket{ip_dst(), dst_port()}; } + // Flags // + + seq_t seq() const noexcept + { return ntohl(tcp_header().seq_nr); } + + seq_t ack() const noexcept + { return ntohl(tcp_header().ack_nr); } + + uint16_t win() const + { return ntohs(tcp_header().window_size); } + + bool isset(Flag f) const noexcept + { return ntohs(tcp_header().offset_flags.whole) & f; } // Get the raw tcp offset, in quadruples - int offset() const + int offset() const noexcept { return tcp_header().offset_flags.offset_reserved >> 4; } // The actual TCP header size (including options). - auto tcp_header_length() const + auto tcp_header_length() const noexcept { return offset() * 4; } + // Data // + + // Where data starts + uint8_t* tcp_data() + { return (uint8_t*)header + tcp_header_length(); } + + const uint8_t* tcp_data() const + { return (uint8_t*)header + tcp_header_length(); } + // Length of data in packet when header has been accounted for - uint16_t tcp_data_length() const + uint16_t tcp_data_length() const noexcept { return ip_data_length() - tcp_header_length(); } + bool has_tcp_data() const noexcept + { return tcp_data_length() > 0; } + + + // Options // + + uint8_t* tcp_options() + { return (uint8_t*) tcp_header().options; } + const uint8_t* tcp_options() const + { return (const uint8_t*) tcp_header().options; } + int tcp_options_length() const + { return tcp_header_length() - sizeof(Header); } + + bool has_tcp_options() const + { return tcp_options_length() > 0; } + + const Option::opt_ts* ts_option() const noexcept + { return ts_opt; } + + inline const Option::opt_ts* parse_ts_option() noexcept; + + + // View operations net::Packet_ptr release() { Expects(pkt != nullptr && "Packet ptr is already null"); return std::move(pkt); } + const net::Packet_ptr& packet_ptr() const noexcept + { return pkt; } + virtual uint16_t compute_tcp_checksum() noexcept = 0; protected: net::Packet_ptr pkt; + + void set_header(uint8_t* hdr) + { Expects(hdr != nullptr); header = reinterpret_cast(hdr); } + +private: Header* header = nullptr; + Option::opt_ts* ts_opt = nullptr; virtual void set_ip_src(const net::Addr& addr) noexcept = 0; virtual void set_ip_dst(const net::Addr& addr) noexcept = 0; @@ -65,4 +123,18 @@ class Packet_view { virtual uint16_t ip_data_length() const noexcept = 0; }; +// assumes the packet contains no other options. +inline const Option::opt_ts* Packet_view::parse_ts_option() noexcept +{ + auto* opt = this->tcp_options(); + // TODO: improve by iterate option instead of byte (see Connection::parse_options) + while(((Option*)opt)->kind == Option::NOP and opt < (uint8_t*)this->tcp_data()) + opt++; + + if(((Option*)opt)->kind == Option::TS) + this->ts_opt = (Option::opt_ts*)opt; + + return this->ts_opt; +} + } diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index 779da354a0..72a2eb3867 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -548,7 +548,7 @@ namespace net { * * @param[in] incoming The incoming tcp packet "to reset". */ - void send_reset(const tcp::Packet& incoming); + void send_reset(const tcp::Packet_view& incoming); /** * @brief Generate a unique initial sequence number (ISS). @@ -576,7 +576,7 @@ namespace net { * * @param[in] A TCP Segment */ - void drop(const tcp::Packet&); + void drop(const tcp::Packet_view&); // INTERNALS - Handling of collections diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 06e6c2d2f9..97c0c539d7 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -281,7 +281,7 @@ void Connection::receive_disconnect() { } } -void Connection::segment_arrived(Packet_ptr incoming) +void Connection::segment_arrived(Packet_view& incoming) { //const uint32_t FMASK = (~(0x0000000F | htons(0x08))); //uint32_t FMASK = 0xFFFFF7F0; @@ -293,7 +293,7 @@ void Connection::segment_arrived(Packet_ptr incoming) // printf("predicted\n"); // Let state handle what to do when incoming packet arrives, and modify the outgoing packet. - switch(state_->handle(*this, std::move(incoming))) + switch(state_->handle(*this, incoming)) { case State::OK: return; // // Do nothing. @@ -368,7 +368,7 @@ void Connection::transmit(Packet_ptr packet) { host_.transmit(std::move(packet)); } -bool Connection::handle_ack(const Packet& in) +bool Connection::handle_ack(const Packet_view& in) { //printf(" RX ACK: %s\n", in.to_string().c_str()); @@ -410,8 +410,9 @@ bool Connection::handle_ack(const Packet& in) if(cb.SND.TS_OK) { - auto* ts = parse_ts_option(in); - last_acked_ts_ = ts->ecr; + const auto* ts = in.ts_option(); + if(ts != nullptr) // TODO: not sure the packet is valid if TS missing + last_acked_ts_ = ts->ecr; } cb.SND.UNA = in.ack(); @@ -438,7 +439,7 @@ bool Connection::handle_ack(const Packet& in) return false; } -void Connection::congestion_control(const Packet& in) +void Connection::congestion_control(const Packet_view& in) { const size_t bytes_acked = highest_ack_ - prev_highest_ack_; @@ -469,7 +470,7 @@ void Connection::congestion_control(const Packet& in) } } -void Connection::fast_recovery(const Packet& in) +void Connection::fast_recovery(const Packet_view& in) { // partial ack /* @@ -550,7 +551,7 @@ void Connection::fast_recovery(const Packet& in) What to do when received ACK is a duplicate. */ -void Connection::on_dup_ack(const Packet& in) +void Connection::on_dup_ack(const Packet_view& in) { // if less than 3 dup acks if(dup_acks_ < 3) @@ -581,7 +582,7 @@ void Connection::on_dup_ack(const Packet& in) // 4.2. Timestamp Heuristic if(cb.SND.TS_OK) { - auto* ts = parse_ts_option(in); + const auto* ts = in.ts_option(); if(ts != nullptr and last_acked_ts_ == ts->ecr) { goto fast_rtx; @@ -679,7 +680,7 @@ void Connection::rtx_ack(const seq_t ack) { This acknowledgment should be piggybacked on a segment being transmitted if possible without incurring undue delay. */ -void Connection::recv_data(const Packet& in) +void Connection::recv_data(const Packet_view& in) { Expects(in.has_tcp_data()); @@ -738,7 +739,7 @@ void Connection::recv_data(const Packet& in) // For now, only full segments are allowed (not partial), // meaning the data will get thrown away if the read buffer not fully fits it. // This makes everything much easier. -void Connection::recv_out_of_order(const Packet& in) +void Connection::recv_out_of_order(const Packet_view& in) { // Packets before this point would totally ruin the buffer Expects((in.seq() - cb.RCV.NXT) < cb.RCV.WND); @@ -832,11 +833,11 @@ void Connection::ack_data() } } -void Connection::take_rtt_measure(const Packet& packet) +void Connection::take_rtt_measure(const Packet_view& packet) { if(cb.SND.TS_OK) { - auto* ts = parse_ts_option(packet); + const auto* ts = packet.ts_option(); if(ts) { rttm.rtt_measurement(RTTM::milliseconds{host_.get_ts_value() - ntohl(ts->ecr)}); @@ -1113,7 +1114,7 @@ std::string Connection::TCB::to_string() const { return std::string(buffer, len); } -void Connection::parse_options(const Packet& packet) { +void Connection::parse_options(const Packet_view& packet) { debug(" Parsing options. Offset: %u, Options: %u \n", packet.offset(), packet.tcp_options_length()); @@ -1273,9 +1274,9 @@ bool Connection::uses_SACK() const noexcept return host_.uses_SACK(); } -void Connection::drop(const Packet& packet, Drop_reason reason) +void Connection::drop(const Packet_view& packet, Drop_reason reason) { - signal_packet_dropped(packet, reason); + signal_packet_dropped(reinterpret_cast(*packet.packet_ptr()), reason); host_.drop(packet); } diff --git a/src/net/tcp/connection_states.cpp b/src/net/tcp/connection_states.cpp index a73f4a77c2..fbf0802334 100644 --- a/src/net/tcp/connection_states.cpp +++ b/src/net/tcp/connection_states.cpp @@ -93,16 +93,16 @@ using namespace std; ///////////////////////////////////////////////////////////////////// // TODO: Optimize this one. It checks for the same things. -bool Connection::State::check_seq(Connection& tcp, const Packet& in) +bool Connection::State::check_seq(Connection& tcp, Packet_view& in) { auto& tcb = tcp.tcb(); uint32_t packet_end = static_cast(in.seq() + in.tcp_data_length()-1); + // RFC 7323 - Option::opt_ts* ts = nullptr; static constexpr uint8_t HEADER_WITH_TS{sizeof(Header) + 12}; if(tcb.SND.TS_OK and in.tcp_header_length() == HEADER_WITH_TS) { - ts = tcp.parse_ts_option(in); + const auto* ts = in.parse_ts_option(); // PAWS if(UNLIKELY(ts != nullptr and (ntohl(ts->val) < tcb.TS_recent and !in.isset(RST)))) @@ -158,6 +158,7 @@ bool Connection::State::check_seq(Connection& tcp, const Packet& in) return false; acceptable: + const auto* ts = in.ts_option(); if(ts != nullptr and (ntohl(ts->val) >= tcb.TS_recent and in.seq() <= tcp.last_ack_sent_)) { @@ -199,7 +200,7 @@ bool Connection::State::check_seq(Connection& tcp, const Packet& in) */ ///////////////////////////////////////////////////////////////////// -void Connection::State::unallowed_syn_reset_connection(Connection& tcp, const Packet& in) { +void Connection::State::unallowed_syn_reset_connection(Connection& tcp, const Packet_view& in) { assert(in.isset(SYN)); debug(" Unallowed SYN for STATE: %s, reseting connection.\n", tcp.state().to_string().c_str()); @@ -223,7 +224,7 @@ void Connection::State::unallowed_syn_reset_connection(Connection& tcp, const Pa */ ///////////////////////////////////////////////////////////////////// -bool Connection::State::check_ack(Connection& tcp, const Packet& in) { +bool Connection::State::check_ack(Connection& tcp, const Packet_view& in) { debug2(" Checking for ACK in STATE: %s \n", tcp.state().to_string().c_str()); if( in.isset(ACK) ) { auto& tcb = tcp.tcb(); @@ -320,7 +321,7 @@ bool Connection::State::check_ack(Connection& tcp, const Packet& in) { */ ///////////////////////////////////////////////////////////////////// -void Connection::State::process_fin(Connection& tcp, const Packet& in) { +void Connection::State::process_fin(Connection& tcp, const Packet_view& in) { debug2(" Processing FIN bit in STATE: %s \n", tcp.state().to_string().c_str()); Expects(in.isset(FIN)); auto& tcb = tcp.tcb(); @@ -677,46 +678,46 @@ void Connection::CloseWait::abort(Connection& tcp) { */ ///////////////////////////////////////////////////////////////////// -State::Result Connection::Closed::handle(Connection& tcp, Packet_ptr in) { - if(in->isset(RST)) { +State::Result Connection::Closed::handle(Connection& tcp, Packet_view& in) { + if(in.isset(RST)) { return OK; } auto packet = tcp.outgoing_packet(); - if(!in->isset(ACK)) { - packet->set_seq(0).set_ack(in->seq() + in->tcp_data_length()).set_flags(RST | ACK); + if(!in.isset(ACK)) { + packet->set_seq(0).set_ack(in.seq() + in.tcp_data_length()).set_flags(RST | ACK); } else { - packet->set_seq(in->ack()).set_flag(RST); + packet->set_seq(in.ack()).set_flag(RST); } tcp.transmit(std::move(packet)); return OK; } -State::Result Connection::Listen::handle(Connection& tcp, Packet_ptr in) { - if(UNLIKELY(in->isset(RST))) +State::Result Connection::Listen::handle(Connection& tcp, Packet_view& in) { + if(UNLIKELY(in.isset(RST))) return OK; - if(UNLIKELY(in->isset(ACK))) + if(UNLIKELY(in.isset(ACK))) { auto packet = tcp.outgoing_packet(); - packet->set_seq(in->ack()).set_flag(RST); + packet->set_seq(in.ack()).set_flag(RST); tcp.transmit(std::move(packet)); return OK; } - if(in->isset(SYN)) + if(in.isset(SYN)) { auto& tcb = tcp.tcb(); - tcb.RCV.NXT = in->seq()+1; - tcb.IRS = in->seq(); + tcb.RCV.NXT = in.seq()+1; + tcb.IRS = in.seq(); tcb.init(); tcb.SND.NXT = tcb.ISS+1; tcb.SND.UNA = tcb.ISS; debug(" Received SYN Packet: %s TCB Updated:\n %s \n", - in->to_string().c_str(), tcp.tcb().to_string().c_str()); + in.to_string().c_str(), tcp.tcb().to_string().c_str()); // Parse options - tcp.parse_options(*in); + tcp.parse_options(in); auto packet = tcp.outgoing_packet(); packet->set_seq(tcb.ISS).set_ack(tcb.RCV.NXT).set_flags(SYN | ACK); @@ -749,23 +750,23 @@ State::Result Connection::Listen::handle(Connection& tcp, Packet_ptr in) { } -State::Result Connection::SynSent::handle(Connection& tcp, Packet_ptr in) { +State::Result Connection::SynSent::handle(Connection& tcp, Packet_view& in) { // 1. check ACK - if(in->isset(ACK)) { + if(in.isset(ACK)) { auto& tcb = tcp.tcb(); // If SEG.ACK =< ISS, or SEG.ACK > SND.NXT - if(UNLIKELY(in->ack() <= tcb.ISS or in->ack() > tcb.SND.NXT)) + if(UNLIKELY(in.ack() <= tcb.ISS or in.ack() > tcb.SND.NXT)) { // send a reset - if(!in->isset(RST)) { + if(!in.isset(RST)) { auto packet = tcp.outgoing_packet(); - packet->set_seq(in->ack()).set_flag(RST); + packet->set_seq(in.ack()).set_flag(RST); tcp.transmit(std::move(packet)); return OK; } // (unless the RST bit is set, if so drop the segment and return) else { - tcp.drop(*in, Drop_reason::RST); + tcp.drop(in, Drop_reason::RST); return OK; } // If SND.UNA =< SEG.ACK =< SND.NXT then the ACK is acceptable. @@ -773,13 +774,13 @@ State::Result Connection::SynSent::handle(Connection& tcp, Packet_ptr in) { } // 2. check RST - if(UNLIKELY(in->isset(RST))) { - if(in->isset(ACK)) { + if(UNLIKELY(in.isset(RST))) { + if(in.isset(ACK)) { tcp.signal_connect(false); - tcp.drop(*in, Drop_reason::RST); + tcp.drop(in, Drop_reason::RST); return CLOSED; } else { - tcp.drop(*in, Drop_reason::RST); + tcp.drop(in, Drop_reason::RST); return OK; } /* @@ -822,27 +823,27 @@ State::Result Connection::SynSent::handle(Connection& tcp, Packet_ptr in) { */ // TODO: Fix this one according to the text above. - if(in->isset(SYN)) { + if(in.isset(SYN)) { auto& tcb = tcp.tcb(); - tcb.RCV.NXT = in->seq()+1; - tcb.IRS = in->seq(); - tcb.SND.UNA = in->ack(); + tcb.RCV.NXT = in.seq()+1; + tcb.IRS = in.seq(); + tcb.SND.UNA = in.ack(); if(tcp.rtx_timer.is_running()) tcp.rtx_stop(); // Parse options - tcp.parse_options(*in); + tcp.parse_options(in); - tcp.take_rtt_measure(*in); + tcp.take_rtt_measure(in); // (our SYN has been ACKed) if(tcb.SND.UNA > tcb.ISS) { // Correction: [RFC 1122 p. 94] - tcb.SND.WND = in->win(); - tcb.SND.WL1 = in->seq(); - tcb.SND.WL2 = in->ack(); + tcb.SND.WND = in.win(); + tcb.SND.WL1 = in.seq(); + tcb.SND.WL2 = in.ack(); // end of correction // [RFC 6298] p.4 (5.7) @@ -865,15 +866,15 @@ State::Result Connection::SynSent::handle(Connection& tcp, Packet_ptr in) { tcp.writeq_push(); // 7. process segment text - if(UNLIKELY(in->has_tcp_data())) + if(UNLIKELY(in.has_tcp_data())) { - tcp.recv_data(*in); + tcp.recv_data(in); } // 8. check FIN bit - if(UNLIKELY(in->isset(FIN))) + if(UNLIKELY(in.isset(FIN))) { - process_fin(tcp, *in); + process_fin(tcp, in); tcp.set_state(Connection::CloseWait::instance()); return OK; } @@ -885,8 +886,8 @@ State::Result Connection::SynSent::handle(Connection& tcp, Packet_ptr in) { packet->set_seq(tcb.ISS).set_ack(tcb.RCV.NXT).set_flags(SYN | ACK); tcp.transmit(std::move(packet)); tcp.set_state(Connection::SynReceived::instance()); - if(in->has_tcp_data()) { - tcp.recv_data(*in); + if(in.has_tcp_data()) { + tcp.recv_data(in); } return OK; /* @@ -898,18 +899,18 @@ State::Result Connection::SynSent::handle(Connection& tcp, Packet_ptr in) { */ } } - tcp.drop(*in); + tcp.drop(in); return OK; } -State::Result Connection::SynReceived::handle(Connection& tcp, Packet_ptr in) { +State::Result Connection::SynReceived::handle(Connection& tcp, Packet_view& in) { // 1. check sequence - if(UNLIKELY(! check_seq(tcp, *in) )) { + if(UNLIKELY(! check_seq(tcp, in) )) { return OK; } // 2. check RST - if(UNLIKELY(in->isset(RST))) { + if(UNLIKELY(in.isset(RST))) { /* If this connection was initiated with a passive OPEN (i.e., came from the LISTEN state), then return this connection to @@ -932,26 +933,26 @@ State::Result Connection::SynReceived::handle(Connection& tcp, Packet_ptr in) { // 3. check security // 4. check SYN - if(UNLIKELY(in->isset(SYN))) + if(UNLIKELY(in.isset(SYN))) { - unallowed_syn_reset_connection(tcp, *in); + unallowed_syn_reset_connection(tcp, in); return CLOSED; } // 5. check ACK - if(in->isset(ACK)) { + if(in.isset(ACK)) { auto& tcb = tcp.tcb(); /* If SND.UNA =< SEG.ACK =< SND.NXT then enter ESTABLISHED state and continue processing. */ - if(tcb.SND.UNA <= in->ack() and in->ack() <= tcb.SND.NXT) + if(tcb.SND.UNA <= in.ack() and in.ack() <= tcb.SND.NXT) { debug2(" SND.UNA =< SEG.ACK =< SND.NXT, continue in ESTABLISHED. \n"); tcp.set_state(Connection::Established::instance()); - tcp.handle_ack(*in); + tcp.handle_ack(in); // [RFC 6298] p.4 (5.7) if(UNLIKELY(tcp.syn_rtx_ > 0)) @@ -963,8 +964,8 @@ State::Result Connection::SynReceived::handle(Connection& tcp, Packet_ptr in) { tcp.signal_connect(); // NOTE: User callback // 7. proccess the segment text - if(UNLIKELY(in->has_tcp_data())) { - tcp.recv_data(*in); + if(UNLIKELY(in.has_tcp_data())) { + tcp.recv_data(in); } } /* @@ -973,19 +974,19 @@ State::Result Connection::SynReceived::handle(Connection& tcp, Packet_ptr in) { */ else { auto packet = tcp.outgoing_packet(); - packet->set_seq(in->ack()).set_flag(RST); + packet->set_seq(in.ack()).set_flag(RST); tcp.transmit(std::move(packet)); } } // ACK is missing else { - tcp.drop(*in, Drop_reason::ACK_NOT_SET); + tcp.drop(in, Drop_reason::ACK_NOT_SET); return OK; } // 8. check FIN - if(UNLIKELY(in->isset(FIN))) { - process_fin(tcp, *in); + if(UNLIKELY(in.isset(FIN))) { + process_fin(tcp, in); tcp.set_state(Connection::CloseWait::instance()); return OK; } @@ -993,14 +994,14 @@ State::Result Connection::SynReceived::handle(Connection& tcp, Packet_ptr in) { } -State::Result Connection::Established::handle(Connection& tcp, Packet_ptr in) { +State::Result Connection::Established::handle(Connection& tcp, Packet_view& in) { // 1. check SEQ - if(UNLIKELY(! check_seq(tcp, *in) )) { + if(UNLIKELY(! check_seq(tcp, in) )) { return OK; } // 2. check RST - if(UNLIKELY( in->isset(RST) )) { + if(UNLIKELY( in.isset(RST) )) { tcp.signal_disconnect(Disconnect::RESET); return CLOSED; // close } @@ -1008,26 +1009,26 @@ State::Result Connection::Established::handle(Connection& tcp, Packet_ptr in) { // 3. check security // 4. check SYN - if(UNLIKELY( in->isset(SYN) )) { - unallowed_syn_reset_connection(tcp, *in); + if(UNLIKELY( in.isset(SYN) )) { + unallowed_syn_reset_connection(tcp, in); return CLOSED; } // 5. check ACK - if(UNLIKELY( ! check_ack(tcp, *in) )) { + if(UNLIKELY( ! check_ack(tcp, in) )) { return OK; } // 6. check URG - DEPRECATED // 7. proccess the segment text - if(in->has_tcp_data()) { - tcp.recv_data(*in); + if(in.has_tcp_data()) { + tcp.recv_data(in); } // 8. check FIN bit - if(UNLIKELY(in->isset(FIN))) { + if(UNLIKELY(in.isset(FIN))) { tcp.set_state(Connection::CloseWait::instance()); - process_fin(tcp, *in); + process_fin(tcp, in); return OK; } @@ -1035,26 +1036,26 @@ State::Result Connection::Established::handle(Connection& tcp, Packet_ptr in) { } -State::Result Connection::FinWait1::handle(Connection& tcp, Packet_ptr in) { +State::Result Connection::FinWait1::handle(Connection& tcp, Packet_view& in) { // 1. Check sequence number - if(UNLIKELY(! check_seq(tcp, *in) )) { + if(UNLIKELY(! check_seq(tcp, in) )) { return OK; } // 2. check RST - if(UNLIKELY( in->isset(RST) )) { + if(UNLIKELY( in.isset(RST) )) { tcp.signal_disconnect(Disconnect::RESET); return CLOSED; // close } // 4. check SYN - if(UNLIKELY( in->isset(SYN) )) { - unallowed_syn_reset_connection(tcp, *in); + if(UNLIKELY( in.isset(SYN) )) { + unallowed_syn_reset_connection(tcp, in); return CLOSED; } // 5. check ACK - if(UNLIKELY( ! check_ack(tcp, *in) )) { + if(UNLIKELY( ! check_ack(tcp, in) )) { return OK; } /* @@ -1063,27 +1064,27 @@ State::Result Connection::FinWait1::handle(Connection& tcp, Packet_ptr in) { processing in that state. */ debug2(" Current TCB:\n %s \n", tcp.tcb().to_string().c_str()); - if(in->ack() == tcp.tcb().SND.NXT) { + if(in.ack() == tcp.tcb().SND.NXT) { // TODO: I guess or FIN is ACK'ed..? tcp.set_state(Connection::FinWait2::instance()); - return tcp.state_->handle(tcp, std::move(in)); // TODO: Is this OK? + return tcp.state_->handle(tcp, in); // TODO: Is this OK? } // 7. proccess the segment text - if(in->has_tcp_data()) { - tcp.recv_data(*in); + if(in.has_tcp_data()) { + tcp.recv_data(in); } // 8. check FIN - if(in->isset(FIN)) { - process_fin(tcp, *in); + if(in.isset(FIN)) { + process_fin(tcp, in); debug2(" FIN isset. TCB:\n %s \n", tcp.tcb().to_string().c_str()); /* If our FIN has been ACKed (perhaps in this segment), then enter TIME-WAIT, start the time-wait timer, turn off the other timers; otherwise enter the CLOSING state. */ - if(in->ack() == tcp.tcb().SND.NXT) { + if(in.ack() == tcp.tcb().SND.NXT) { // TODO: I guess or FIN is ACK'ed..? tcp.set_state(TimeWait::instance()); if(tcp.rtx_timer.is_running()) @@ -1097,37 +1098,37 @@ State::Result Connection::FinWait1::handle(Connection& tcp, Packet_ptr in) { } -State::Result Connection::FinWait2::handle(Connection& tcp, Packet_ptr in) { +State::Result Connection::FinWait2::handle(Connection& tcp, Packet_view& in) { // 1. check SEQ - if(! check_seq(tcp, *in) ) { + if(! check_seq(tcp, in) ) { return OK; } // 2. check RST - if( in->isset(RST) ) { + if( in.isset(RST) ) { tcp.signal_disconnect(Disconnect::RESET); return CLOSED; // close } // 4. check SYN - if( in->isset(SYN) ) { - unallowed_syn_reset_connection(tcp, *in); + if( in.isset(SYN) ) { + unallowed_syn_reset_connection(tcp, in); return CLOSED; } // 5. check ACK - if( ! check_ack(tcp, *in) ) { + if( ! check_ack(tcp, in) ) { return OK; } // 7. proccess the segment text - if(in->has_tcp_data()) { - tcp.recv_data(*in); + if(in.has_tcp_data()) { + tcp.recv_data(in); } // 8. check FIN - if(in->isset(FIN)) { - process_fin(tcp, *in); + if(in.isset(FIN)) { + process_fin(tcp, in); /* Enter the TIME-WAIT state. Start the time-wait timer, turn off the other timers. @@ -1141,26 +1142,26 @@ State::Result Connection::FinWait2::handle(Connection& tcp, Packet_ptr in) { } -State::Result Connection::CloseWait::handle(Connection& tcp, Packet_ptr in) { +State::Result Connection::CloseWait::handle(Connection& tcp, Packet_view& in) { // 1. check SEQ - if(! check_seq(tcp, *in) ) { + if(! check_seq(tcp, in) ) { return OK; } // 2. check RST - if( in->isset(RST) ) { + if( in.isset(RST) ) { tcp.signal_disconnect(Disconnect::RESET); return CLOSED; // close } // 4. check SYN - if( in->isset(SYN) ) { - unallowed_syn_reset_connection(tcp, *in); + if( in.isset(SYN) ) { + unallowed_syn_reset_connection(tcp, in); return CLOSED; } // 5. check ACK - if( ! check_ack(tcp, *in) ) { + if( ! check_ack(tcp, in) ) { return OK; } @@ -1168,8 +1169,8 @@ State::Result Connection::CloseWait::handle(Connection& tcp, Packet_ptr in) { // This should not occur, since a FIN has been received from the remote side. Ignore the segment text. // 8. check FIN - if(in->isset(FIN)) { - process_fin(tcp, *in); + if(in.isset(FIN)) { + process_fin(tcp, in); // Remain in state return OK; } @@ -1177,25 +1178,25 @@ State::Result Connection::CloseWait::handle(Connection& tcp, Packet_ptr in) { } -State::Result Connection::Closing::handle(Connection& tcp, Packet_ptr in) { +State::Result Connection::Closing::handle(Connection& tcp, Packet_view& in) { // 1. Check sequence number - if(! check_seq(tcp, *in) ) { + if(! check_seq(tcp, in) ) { return OK; } // 2. check RST - if( in->isset(RST) ) { + if( in.isset(RST) ) { return CLOSED; // close } // 4. check SYN - if( in->isset(SYN) ) { - unallowed_syn_reset_connection(tcp, *in); + if( in.isset(SYN) ) { + unallowed_syn_reset_connection(tcp, in); return CLOSED; } // 5. check ACK - if( ! check_ack(tcp, *in)) { + if( ! check_ack(tcp, in)) { return CLOSED; } @@ -1204,7 +1205,7 @@ State::Result Connection::Closing::handle(Connection& tcp, Packet_ptr in) { the ACK acknowledges our FIN then enter the TIME-WAIT state, otherwise ignore the segment. */ - if(in->ack() == tcp.tcb().SND.NXT) { + if(in.ack() == tcp.tcb().SND.NXT) { // TODO: I guess or FIN is ACK'ed..? tcp.set_state(TimeWait::instance()); tcp.timewait_start(); @@ -1214,8 +1215,8 @@ State::Result Connection::Closing::handle(Connection& tcp, Packet_ptr in) { // This should not occur, since a FIN has been received from the remote side. Ignore the segment text. // 8. check FIN - if(in->isset(FIN)) { - process_fin(tcp, *in); + if(in.isset(FIN)) { + process_fin(tcp, in); // Remain in state return OK; } @@ -1223,25 +1224,25 @@ State::Result Connection::Closing::handle(Connection& tcp, Packet_ptr in) { } -State::Result Connection::LastAck::handle(Connection& tcp, Packet_ptr in) { +State::Result Connection::LastAck::handle(Connection& tcp, Packet_view& in) { // 1. Check sequence number - if(! check_seq(tcp, *in) ) { + if(! check_seq(tcp, in) ) { return OK; } return CLOSED; // 2. check RST - if( in->isset(RST) ) { + if( in.isset(RST) ) { return CLOSED; // close } // 4. check SYN - if( in->isset(SYN) ) { - unallowed_syn_reset_connection(tcp, *in); + if( in.isset(SYN) ) { + unallowed_syn_reset_connection(tcp, in); return CLOSED; } - if( ! check_ack(tcp, *in)) { + if( ! check_ack(tcp, in)) { return CLOSED; } @@ -1249,8 +1250,8 @@ State::Result Connection::LastAck::handle(Connection& tcp, Packet_ptr in) { // This should not occur, since a FIN has been received from the remote side. Ignore the segment text. // 8. check FIN - if(in->isset(FIN)) { - process_fin(tcp, *in); + if(in.isset(FIN)) { + process_fin(tcp, in); // Remain in state return OK; } @@ -1258,20 +1259,20 @@ State::Result Connection::LastAck::handle(Connection& tcp, Packet_ptr in) { } -State::Result Connection::TimeWait::handle(Connection& tcp, Packet_ptr in) { +State::Result Connection::TimeWait::handle(Connection& tcp, Packet_view& in) { // 1. Check sequence number - if(! check_seq(tcp, *in) ) { + if(! check_seq(tcp, in) ) { return OK; } // 2. check RST - if( in->isset(RST) ) { + if( in.isset(RST) ) { return CLOSED; // close } // 4. check SYN - if( in->isset(SYN) ) { - unallowed_syn_reset_connection(tcp, *in); + if( in.isset(SYN) ) { + unallowed_syn_reset_connection(tcp, in); return CLOSED; } @@ -1279,8 +1280,8 @@ State::Result Connection::TimeWait::handle(Connection& tcp, Packet_ptr in) { // This should not occur, since a FIN has been received from the remote side. Ignore the segment text. // 8. check FIN - if(in->isset(FIN)) { - process_fin(tcp, *in); + if(in.isset(FIN)) { + process_fin(tcp, in); // Remain in state tcp.timewait_restart(); return OK; diff --git a/src/net/tcp/listener.cpp b/src/net/tcp/listener.cpp index d87a30d1ab..dad0f7fc0b 100644 --- a/src/net/tcp/listener.cpp +++ b/src/net/tcp/listener.cpp @@ -39,14 +39,13 @@ bool Listener::syn_queue_full() const { return syn_queue_.size() >= host_.max_syn_backlog(); } -void Listener::segment_arrived(Packet_ptr packet) { +void Listener::segment_arrived(Packet_view& packet) { debug2(" Received packet: %s\n", - packet->to_string().c_str()); + packet.to_string().c_str()); auto it = std::find_if(syn_queue_.begin(), syn_queue_.end(), - [dest = packet->source()] - (Connection_ptr conn) { - return conn->remote() == dest; + [&packet] (const Connection_ptr& conn) { + return conn->remote() == packet.source(); }); // if it's an reply to any of our half-open connections @@ -55,7 +54,7 @@ void Listener::segment_arrived(Packet_ptr packet) { auto conn = *it; debug(" Found packet receiver: %s\n", conn->to_string().c_str()); - conn->segment_arrived(std::move(packet)); + conn->segment_arrived(packet); debug2(" Connection done handling segment\n"); return; } @@ -63,9 +62,9 @@ void Listener::segment_arrived(Packet_ptr packet) { else { // don't waste time if the packet does not have SYN - if(UNLIKELY(not packet->isset(SYN) or packet->has_tcp_data())) + if(UNLIKELY(not packet.isset(SYN) or packet.has_tcp_data())) { - host_.send_reset(*packet); + host_.send_reset(packet); return; } @@ -73,7 +72,7 @@ void Listener::segment_arrived(Packet_ptr packet) { host_.connection_attempts_++; // if we don't like this client, do nothing - if(UNLIKELY(on_accept_(packet->source()) == false)) + if(UNLIKELY(on_accept_(packet.source()) == false)) return; // remove oldest connection if queue is full @@ -91,7 +90,7 @@ void Listener::segment_arrived(Packet_ptr packet) { auto& conn = *(syn_queue_.emplace( syn_queue_.cbegin(), - std::make_shared(host_, packet->destination(), packet->source(), ConnectCallback{this, &Listener::connected}) + std::make_shared(host_, packet.destination(), packet.source(), ConnectCallback{this, &Listener::connected}) ) ); conn->_on_cleanup({this, &Listener::remove}); @@ -100,7 +99,7 @@ void Listener::segment_arrived(Packet_ptr packet) { Ensures(conn->is_listening()); debug(" Connection %s created\n", conn->to_string().c_str()); - conn->segment_arrived(std::move(packet)); + conn->segment_arrived(packet); debug2(" Connection done handling segment\n"); return; } diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index 00ba9398d0..d81c444ef6 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -207,17 +207,15 @@ void TCP::receive(Packet_view& packet) // validate some unlikely but invalid packet properties if (UNLIKELY(packet.src_port() == 0)) { - drop(*static_unique_ptr_cast(packet.release())); + drop(packet); return; } - const auto dest = packet.destination(); - // Validate checksum if (UNLIKELY(packet.compute_tcp_checksum() != 0)) { PRINT(" TCP Packet Checksum %#x != %#x\n", packet.compute_tcp_checksum(), 0x0); - drop(*static_unique_ptr_cast(packet.release())); + drop(packet); return; } @@ -230,6 +228,7 @@ void TCP::receive(Packet_view& packet) return; } + const auto dest = packet.destination(); const Connection::Tuple tuple { dest, packet.source() }; // Try to find the receiver @@ -238,7 +237,7 @@ void TCP::receive(Packet_view& packet) // Connection found if (conn_it != connections_.end()) { PRINT(" Connection found: %s \n", conn_it->second->to_string().c_str()); - conn_it->second->segment_arrived(static_unique_ptr_cast(packet.release())); + conn_it->second->segment_arrived(packet); return; } @@ -250,16 +249,15 @@ void TCP::receive(Packet_view& packet) if (listener_it != listeners_.end()) { auto& listener = listener_it->second; PRINT(" Listener found: %s\n", listener->to_string().c_str()); - listener->segment_arrived(static_unique_ptr_cast(packet.release())); + listener->segment_arrived(packet); PRINT(" Listener done with packet\n"); return; } - auto pkt = static_unique_ptr_cast(packet.release()); // Send a reset - send_reset(*pkt); + send_reset(packet); - drop(*pkt); + drop(packet); } // Show all connections for TCP as a string. @@ -380,7 +378,7 @@ tcp::Packet_ptr TCP::create_outgoing_packet() return packet; } -void TCP::send_reset(const tcp::Packet& in) +void TCP::send_reset(const tcp::Packet_view& in) { // TODO: maybe worth to just swap the fields in // the incoming packet and send that one @@ -404,7 +402,7 @@ uint32_t TCP::get_ts_value() const return ((RTC::nanos_now() / 1000000000ull) & 0xffffffff); } -void TCP::drop(const tcp::Packet&) { +void TCP::drop(const tcp::Packet_view&) { // Stat increment packets dropped (*packets_dropped_)++; debug(" Packet dropped\n"); From 8f057042d4f19e76c3abd473788de96e931046f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 7 Jun 2018 15:33:50 +0200 Subject: [PATCH 400/723] tcp: Prepare creation of tcp v6 packets --- api/net/tcp/common.hpp | 3 + api/net/tcp/connection.hpp | 9 +- api/net/tcp/packet.hpp | 2 +- api/net/tcp/packet4_view.hpp | 29 +++++- api/net/tcp/packet6_view.hpp | 16 ++- api/net/tcp/packet_view.hpp | 189 ++++++++++++++++++++++++++++++----- api/net/tcp/tcp.hpp | 9 +- src/net/tcp/connection.cpp | 11 +- src/net/tcp/tcp.cpp | 16 ++- 9 files changed, 244 insertions(+), 40 deletions(-) diff --git a/api/net/tcp/common.hpp b/api/net/tcp/common.hpp index e1a6a80d00..4f2494b80b 100644 --- a/api/net/tcp/common.hpp +++ b/api/net/tcp/common.hpp @@ -68,6 +68,9 @@ namespace net { class Packet; using Packet_ptr = std::unique_ptr; + class Packet_view; + using Packet_view_ptr = std::unique_ptr; + class Connection; using Connection_ptr = std::shared_ptr; diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index d6cbc80a6f..e7106fa174 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -422,7 +422,7 @@ class Connection { * * @return A TCP Socket */ - Socket local() const noexcept + const Socket& local() const noexcept { return local_; } /** @@ -430,9 +430,12 @@ class Connection { * * @return A TCP Socket */ - Socket remote() const noexcept + const Socket& remote() const noexcept { return remote_; } + Protocol ipv() const noexcept + { return is_ipv6_ ? Protocol::IPv6 : Protocol::IPv4; } + auto bytes_sacked() const noexcept { return bytes_sacked_; } @@ -626,6 +629,8 @@ class Connection { Socket local_; Socket remote_; + const bool is_ipv6_ = false; + /** The current state the Connection is in. Handles most of the logic. */ State* state_; // Previous state. Used to keep track of state transitions. diff --git a/api/net/tcp/packet.hpp b/api/net/tcp/packet.hpp index 3dabd74dbf..dc80751ad4 100644 --- a/api/net/tcp/packet.hpp +++ b/api/net/tcp/packet.hpp @@ -48,7 +48,7 @@ class Packet : public PacketIP4 { //! a valid MTU-sized buffer void init() { - PacketIP4::init(Protocol::TCP); + //PacketIP4::init(Protocol::TCP); auto* ipdata = ip_data_ptr(); // clear TCP header diff --git a/api/net/tcp/packet4_view.hpp b/api/net/tcp/packet4_view.hpp index 5bd6a987e3..c309e1cfd8 100644 --- a/api/net/tcp/packet4_view.hpp +++ b/api/net/tcp/packet4_view.hpp @@ -15,9 +15,14 @@ class Packet4_view : public Packet_view { set_header(packet().ip_data().data()); } - uint16_t compute_tcp_checksum() noexcept override + inline void init(); + + uint16_t compute_tcp_checksum() const noexcept override { return calculate_checksum6(*this); } + Protocol ipv() const noexcept override + { return Protocol::IPv4; } + private: PacketIP4& packet() noexcept { return static_cast(*pkt); } @@ -37,8 +42,26 @@ class Packet4_view : public Packet_view { net::Addr ip_dst() const noexcept override { return packet().ip_dst(); } - uint16_t ip_data_length() const noexcept override - { return packet().ip_data_length(); } + uint16_t ip_data_length() const noexcept override + { return packet().ip_data_length(); } + + uint16_t ip_header_length() const noexcept override + { return packet().ip_header_length(); } }; +inline void Packet4_view::init() +{ + //memset(header, 0, sizeof(tcp::Header)); + auto* ipdata = packet().ip_data().data(); + + // clear TCP header + ((uint32_t*) ipdata)[3] = 0; + ((uint32_t*) ipdata)[4] = 0; + + set_win(tcp::default_window_size); + header->offset_flags.offset_reserved = (5 << 4); + + set_length(); +} + } diff --git a/api/net/tcp/packet6_view.hpp b/api/net/tcp/packet6_view.hpp index c45154ab2d..3d1eeda051 100644 --- a/api/net/tcp/packet6_view.hpp +++ b/api/net/tcp/packet6_view.hpp @@ -15,9 +15,14 @@ class Packet6_view : public Packet_view { set_header(packet().ip_data().data()); } - uint16_t compute_tcp_checksum() noexcept override + inline void init(); + + uint16_t compute_tcp_checksum() const noexcept override { return calculate_checksum6(*this); } + Protocol ipv() const noexcept override + { return Protocol::IPv4; } + private: PacketIP6& packet() noexcept { return static_cast(*pkt); } @@ -39,6 +44,15 @@ class Packet6_view : public Packet_view { uint16_t ip_data_length() const noexcept override { return packet().ip_data_length(); } + + uint16_t ip_header_length() const noexcept override + { return packet().ip_header_len(); } }; + +inline void Packet6_view::init() +{ + set_length(); +} + } diff --git a/api/net/tcp/packet_view.hpp b/api/net/tcp/packet_view.hpp index 4a795e7854..0a14f25eae 100644 --- a/api/net/tcp/packet_view.hpp +++ b/api/net/tcp/packet_view.hpp @@ -8,11 +8,6 @@ namespace net::tcp { class Packet_view { public: - Packet_view(net::Packet_ptr ptr) - : pkt{std::move(ptr)} - { - Expects(pkt != nullptr); - } Header& tcp_header() noexcept { return *header; } @@ -34,6 +29,26 @@ class Packet_view { Socket destination() const noexcept { return Socket{ip_dst(), dst_port()}; } + Packet_view& set_src_port(port_t p) noexcept + { tcp_header().source_port = htons(p); return *this; } + + Packet_view& set_dst_port(port_t p) noexcept + { tcp_header().destination_port = htons(p); return *this; } + + Packet_view& set_source(const Socket& src) + { + set_ip_src(src.address()); + set_src_port(src.port()); + return *this; + } + + Packet_view& set_destination(const Socket& dest) + { + set_ip_dst(dest.address()); + set_dst_port(dest.port()); + return *this; + } + // Flags // seq_t seq() const noexcept @@ -42,7 +57,7 @@ class Packet_view { seq_t ack() const noexcept { return ntohl(tcp_header().ack_nr); } - uint16_t win() const + uint16_t win() const noexcept { return ntohs(tcp_header().window_size); } bool isset(Flag f) const noexcept @@ -52,26 +67,47 @@ class Packet_view { int offset() const noexcept { return tcp_header().offset_flags.offset_reserved >> 4; } - // The actual TCP header size (including options). - auto tcp_header_length() const noexcept - { return offset() * 4; } - // Data // + Packet_view& set_seq(seq_t n) noexcept + { tcp_header().seq_nr = htonl(n); return *this; } - // Where data starts - uint8_t* tcp_data() - { return (uint8_t*)header + tcp_header_length(); } + Packet_view& set_ack(seq_t n) noexcept + { tcp_header().ack_nr = htonl(n); return *this; } - const uint8_t* tcp_data() const - { return (uint8_t*)header + tcp_header_length(); } + Packet_view& set_win(uint16_t size) noexcept + { tcp_header().window_size = htons(size); return *this; } - // Length of data in packet when header has been accounted for - uint16_t tcp_data_length() const noexcept - { return ip_data_length() - tcp_header_length(); } + Packet_view& set_flag(Flag f) noexcept + { tcp_header().offset_flags.whole |= htons(f); return *this; } + + Packet_view& set_flags(uint16_t f) noexcept + { tcp_header().offset_flags.whole |= htons(f); return *this; } + + Packet_view& clear_flag(Flag f) noexcept + { tcp_header().offset_flags.whole &= ~ htons(f); return *this; } + + Packet_view& clear_flags() + { tcp_header().offset_flags.whole &= 0x00ff; return *this; } + + // Set raw TCP offset in quadruples + void set_offset(int offset) + { tcp_header().offset_flags.offset_reserved = (offset & 0xF) << 4; } - bool has_tcp_data() const noexcept - { return tcp_data_length() > 0; } + // The actual TCP header size (including options). + auto tcp_header_length() const noexcept + { return offset() * 4; } + + virtual uint16_t compute_tcp_checksum() const noexcept = 0; + + Packet_view& set_tcp_checksum(uint16_t checksum) noexcept + { tcp_header().checksum = checksum; return *this; } + + void set_tcp_checksum() noexcept + { + tcp_header().checksum = 0; + set_tcp_checksum(compute_tcp_checksum()); + } // Options // @@ -87,11 +123,49 @@ class Packet_view { bool has_tcp_options() const { return tcp_options_length() > 0; } + /** + * @brief Adds a tcp option. + * + * @todo It's probably a better idea to make the option include + * the padding for it to be aligned, and avoid two mem operations + * + * @tparam T TCP Option + * @tparam Padding padding in bytes to be put infront of the option + * @tparam Args construction args to option T + */ + template + void add_tcp_option(Args&&... args); + + /** + * @brief Adds a tcp option aligned. + * Assumes the user knows what she/he is doing. + * + * @tparam T An aligned TCP option + * @tparam Args construction args to option T + */ + template + void add_tcp_option_aligned(Args&&... args); + + const Option::opt_ts* ts_option() const noexcept { return ts_opt; } inline const Option::opt_ts* parse_ts_option() noexcept; + // Data // + + uint8_t* tcp_data() + { return (uint8_t*)header + tcp_header_length(); } + + const uint8_t* tcp_data() const + { return (uint8_t*)header + tcp_header_length(); } + + // Length of data in packet when header has been accounted for + uint16_t tcp_data_length() const noexcept + { return ip_data_length() - tcp_header_length(); } + + bool has_tcp_data() const noexcept + { return tcp_data_length() > 0; } // View operations net::Packet_ptr release() @@ -103,26 +177,95 @@ class Packet_view { const net::Packet_ptr& packet_ptr() const noexcept { return pkt; } - virtual uint16_t compute_tcp_checksum() noexcept = 0; + // hmm + virtual Protocol ipv() const noexcept = 0; + + virtual ~Packet_view() = default; + protected: net::Packet_ptr pkt; + Header* header = nullptr; + + Packet_view(net::Packet_ptr ptr) + : pkt{std::move(ptr)} + { + Expects(pkt != nullptr); + } void set_header(uint8_t* hdr) { Expects(hdr != nullptr); header = reinterpret_cast(hdr); } + // sets the correct length for all the protocols up to IP4 + void set_length(uint16_t newlen = 0) + { pkt->set_data_end(ip_header_length() + tcp_header_length() + newlen); } + private: - Header* header = nullptr; Option::opt_ts* ts_opt = nullptr; virtual void set_ip_src(const net::Addr& addr) noexcept = 0; virtual void set_ip_dst(const net::Addr& addr) noexcept = 0; virtual net::Addr ip_src() const noexcept = 0; virtual net::Addr ip_dst() const noexcept = 0; - // TODO: see if we can get rid of this virtual call + + // TODO: see if we can get rid of these virtual calls virtual uint16_t ip_data_length() const noexcept = 0; + virtual uint16_t ip_header_length() const noexcept = 0; }; + +template +inline void Packet_view::add_tcp_option(Args&&... args) { + // to avoid headache, options need to be added BEFORE any data. + assert(!has_tcp_data()); + struct NOP { + uint8_t kind{0x01}; + }; + // option address + auto* addr = tcp_options()+tcp_options_length(); + // if to use pre padding + if(Padding) + new (addr) NOP[Padding]; + + // emplace the option after pre padding + const auto& opt = *(new (addr + Padding) T(args...)); + + // find number of NOP to pad with + const auto nops = (opt.length + Padding) % 4; + if(nops) { + new (addr + Padding + opt.length) NOP[nops]; + } + + // update offset + auto newoffset = offset() + round_up(opt.length + Padding, 4); + if (UNLIKELY(newoffset > 0xF)) { + throw std::runtime_error("Too many TCP options"); + } + set_offset(newoffset); + + set_length(); // update +} + +template +inline void Packet_view::add_tcp_option_aligned(Args&&... args) { + // to avoid headache, options need to be added BEFORE any data. + Expects(!has_tcp_data()); + + // option address + auto* addr = tcp_options()+tcp_options_length(); + // emplace the option + auto& opt = *(new (addr) T(args...)); + + // update offset + auto newoffset = offset() + round_up(opt.size(), 4); + + set_offset(newoffset); + if (UNLIKELY(newoffset > 0xF)) { + throw std::runtime_error("Too many TCP options"); + } + set_length(); // update +} + // assumes the packet contains no other options. inline const Option::opt_ts* Packet_view::parse_ts_option() noexcept { diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index 72a2eb3867..7cc7c57fb2 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -540,7 +540,14 @@ namespace net { * * @return A tcp packet ptr */ - tcp::Packet_ptr create_outgoing_packet(); + tcp::Packet_view_ptr create_outgoing_packet(); + + /** + * @brief Creates an outgoing TCP6 packet. + * + * @return A tcp packet ptr + */ + tcp::Packet_view_ptr create_outgoing_packet6(); /** * @brief Sends a TCP reset based on the values of the incoming packet. diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index 97c0c539d7..f59fe4e9c4 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -30,8 +30,8 @@ using namespace std; Connection::Connection(TCP& host, Socket local, Socket remote, ConnectCallback callback) : host_(host), - local_(local), - remote_(remote), + local_{std::move(local)}, remote_{std::move(remote)}, + is_ipv6_(local_.address().is_v6()), state_(&Connection::Closed::instance()), prev_state_(state_), cb{host_.window_size()}, @@ -318,9 +318,10 @@ int Connection::serialize_to(void*) const { return 0; } Packet_ptr Connection::create_outgoing_packet() { - auto packet = host_.create_outgoing_packet(); + auto packet = (is_ipv6_) ? + host_.create_outgoing_packet6() : host_.create_outgoing_packet(); // Set Source (local == the current connection) - packet->set_source(local()); + packet->set_source(local_); // Set Destination (remote) packet->set_destination(remote_); @@ -346,7 +347,7 @@ Packet_ptr Connection::create_outgoing_packet() packet->set_seq(cb.SND.NXT).set_ack(cb.RCV.NXT); debug(" Outgoing packet created: %s \n", packet->to_string().c_str()); - return packet; + return static_unique_ptr_cast(packet->release()); } void Connection::transmit(Packet_ptr packet) { diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index d81c444ef6..5af371ab55 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -371,9 +371,16 @@ void TCP::transmit(tcp::Packet_ptr packet) { _network_layer_out(std::move(packet)); } -tcp::Packet_ptr TCP::create_outgoing_packet() +tcp::Packet_view_ptr TCP::create_outgoing_packet() { - auto packet = static_unique_ptr_cast(inet_.create_packet()); + auto packet = std::make_unique(inet_.create_ip_packet(Protocol::TCP)); + packet->init(); + return packet; +} + +tcp::Packet_view_ptr TCP::create_outgoing_packet6() +{ + auto packet = std::make_unique(inet_.create_ip6_packet(Protocol::TCP)); packet->init(); return packet; } @@ -382,14 +389,15 @@ void TCP::send_reset(const tcp::Packet_view& in) { // TODO: maybe worth to just swap the fields in // the incoming packet and send that one - auto out = create_outgoing_packet(); + auto out = (in.ipv() == Protocol::IPv6) + ? create_outgoing_packet6() : create_outgoing_packet(); // increase incoming SEQ and ACK by 1 and set RST + ACK out->set_seq(in.ack()+1).set_ack(in.seq()+1).set_flags(RST | ACK); // swap dest and src out->set_source(in.destination()); out->set_destination(in.source()); - transmit(std::move(out)); + transmit(static_unique_ptr_cast(out->release())); } seq_t TCP::generate_iss() { From c377a8d57266827cf486bee6047d6205397bf23c Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 7 Jun 2018 17:53:31 +0200 Subject: [PATCH 401/723] sha1: Accept const void* for update() --- api/util/sha1.hpp | 2 +- src/util/sha1.cpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/util/sha1.hpp b/api/util/sha1.hpp index d27f11fde4..d2e2297544 100644 --- a/api/util/sha1.hpp +++ b/api/util/sha1.hpp @@ -36,7 +36,7 @@ class SHA1 // update with new data void update(const std::string&); void update(const std::vector&); - void update(const char*, size_t); + void update(const void*, size_t); // finalize values std::vector as_raw(); // 20 bytes std::string as_hex(); // 40 bytes diff --git a/src/util/sha1.cpp b/src/util/sha1.cpp index f2f77c7810..d98598fb1f 100644 --- a/src/util/sha1.cpp +++ b/src/util/sha1.cpp @@ -230,8 +230,9 @@ void SHA1::update(const std::vector& vec) update(vec.data(), vec.size()); } -void SHA1::update(const char* inbuffer, size_t inlen) +void SHA1::update(const void* inbuffer_v, size_t inlen) { + auto* inbuffer = (const char*) inbuffer_v; while (inlen) { // find out how much we can copy From 41cea05d40302d9cb0ab6acf14156ecb3c04bc20 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 7 Jun 2018 17:56:37 +0200 Subject: [PATCH 402/723] test: Add unittest for FAT --- test/CMakeLists.txt | 3 + test/fs/unit/unit_fat.cpp | 145 +++++++++++++++++++++++++++++++++++++ test/fs/unit/unit_fs.cpp | 10 +++ test/lest_util/mock_fs.cpp | 47 ++++++++++++ test/lest_util/mock_fs.hpp | 34 +++++++++ test/memdisk.fat | Bin 0 -> 14336 bytes 6 files changed, 239 insertions(+) create mode 100644 test/fs/unit/unit_fat.cpp create mode 100644 test/fs/unit/unit_fs.cpp create mode 100644 test/lest_util/mock_fs.cpp create mode 100644 test/lest_util/mock_fs.hpp create mode 100644 test/memdisk.fat diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index abba07a694..7a33ad92e9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -78,6 +78,7 @@ include_directories( set(LEST_UTIL ${TEST}/lest_util/lestmain.cxx ${TEST}/lest_util/os_mock.cpp + ${TEST}/lest_util/mock_fs.cpp ${TEST}/lest_util/random.cpp ) @@ -86,6 +87,8 @@ set(TEST_SOURCES ${TEST}/fs/unit/memdisk_test.cpp ${TEST}/fs/unit/path_test.cpp ${TEST}/fs/unit/vfs_test.cpp + ${TEST}/fs/unit/unit_fs.cpp + ${TEST}/fs/unit/unit_fat.cpp #${TEST}/hw/unit/cpu_test.cpp ${TEST}/hw/unit/mac_addr_test.cpp ${TEST}/hw/unit/nic_test.cpp diff --git a/test/fs/unit/unit_fat.cpp b/test/fs/unit/unit_fat.cpp new file mode 100644 index 0000000000..1002db50ea --- /dev/null +++ b/test/fs/unit/unit_fat.cpp @@ -0,0 +1,145 @@ +#include +#include +#include +#include +using namespace fs; + +static MemDisk* mdisk = nullptr; +static Disk_ptr disk = nullptr; + +CASE("Prepare custom memdisk") +{ + auto* fp = fopen("../memdisk.fat", "rb"); + EXPECT(fp != nullptr); + fseek(fp, 0L, SEEK_END); + long int size = ftell(fp); + rewind(fp); + // read file into buffer + char* buffer = new char[size]; + EXPECT(buffer); + size_t res = fread(buffer, size, 1, fp); + EXPECT(res == 1); + // create memdisk + mdisk = new MemDisk(buffer, buffer + size); + EXPECT(mdisk); +} + +CASE("Initialize FAT fs") +{ + disk = std::make_shared (*mdisk); + disk->init_fs( + [&lest_env] (auto err, File_system& fs) + { + EXPECT(!err); + }); +} + +CASE("List root directory") +{ + auto& fs = disk->fs(); + auto res = fs.ls("/"); + // validate root directory + EXPECT(!res.error); + EXPECT(res.entries->size() == 8); + + // validate each entry + auto& ents = *res.entries; + EXPECT(ents.at(0).name() == "."); + EXPECT(ents.at(1).name() == ".."); + EXPECT(ents.at(2).name() == "folder"); + EXPECT(ents.at(3).name() == "test.pem"); + EXPECT(ents.at(4).name() == "test.der"); + EXPECT(ents.at(5).name() == "build.sh"); + EXPECT(ents.at(6).name() == "server.key"); + EXPECT(ents.at(7).name() == "test.key"); +} + +CASE("Validate /folder") +{ + auto& fs = disk->fs(); + // validate folder sync + auto res = fs.ls("/folder/"); + EXPECT(!res.error); + EXPECT(res.entries->size() == 3); + + // validate each entry + auto& ents = *res.entries; + EXPECT(ents.at(0).name() == "."); + EXPECT(ents.at(1).name() == ".."); + EXPECT(ents.at(2).name() == "file.txt"); + + // validate folder async + fs.ls("/folder", + [&lest_env] (auto error, Dirvec_ptr dirvec) + { + EXPECT(!error); + auto& ents = *dirvec; + EXPECT(ents.at(0).name() == "."); + EXPECT(ents.at(1).name() == ".."); + EXPECT(ents.at(2).name() == "file.txt"); + }); +} + +CASE("Validate /folder using dirent") +{ + auto& fs = disk->fs(); + // get dirent by using stat() + auto ent = fs.stat("/folder"); + EXPECT(ent.is_valid()); + EXPECT(ent.is_dir()); + + // validate folder sync using dirent + auto res = fs.ls(ent); + EXPECT(!res.error); + EXPECT(res.entries->size() == 3); + + // validate each entry + auto& ents = *res.entries; + EXPECT(ents.at(0).name() == "."); + EXPECT(ents.at(1).name() == ".."); + EXPECT(ents.at(2).name() == "file.txt"); + + // validate folder async using dirent + fs.ls(ent, + [&lest_env] (auto error, Dirvec_ptr dirvec) + { + EXPECT(!error); + auto& ents = *dirvec; + EXPECT(ents.at(0).name() == "."); + EXPECT(ents.at(1).name() == ".."); + EXPECT(ents.at(2).name() == "file.txt"); + }); +} + +CASE("Validate /folder/file.txt") +{ + auto& fs = disk->fs(); + // read and validate file + auto buffer = fs.read_file("/folder/file.txt"); + EXPECT(buffer.size() == 24); + SHA1 sha1; + sha1.update(buffer.data(), buffer.size()); + const std::string hash = sha1.as_hex(); + + EXPECT(hash == "de1d4db81aa9344377ccfd49f5a239533d4f4ee3"); + + const std::string text((const char*) buffer.data(), buffer.size()); + EXPECT(text == "This file contains text\n"); +} + +CASE("Stat /folder/file.txt") +{ + auto& fs = disk->fs(); + // get dirent by stat() + auto ent = fs.stat("/folder/file.txt"); + EXPECT(ent.is_valid()); + EXPECT(ent.is_file()); + EXPECT(ent.size() == 24); + + // read and validate file using dirent + auto buffer = fs.read(ent, 0, ent.size()); + EXPECT(buffer.size() == 24); + + const std::string text((const char*) buffer.data(), buffer.size()); + EXPECT(text == "This file contains text\n"); +} diff --git a/test/fs/unit/unit_fs.cpp b/test/fs/unit/unit_fs.cpp new file mode 100644 index 0000000000..b5eadc1ca2 --- /dev/null +++ b/test/fs/unit/unit_fs.cpp @@ -0,0 +1,10 @@ +#include +#include + +using namespace fs; + +CASE("Initialize mock FS") +{ + MockFS fs; + +} diff --git a/test/lest_util/mock_fs.cpp b/test/lest_util/mock_fs.cpp new file mode 100644 index 0000000000..0fec20b323 --- /dev/null +++ b/test/lest_util/mock_fs.cpp @@ -0,0 +1,47 @@ +#include "mock_fs.hpp" + +namespace fs +{ + void MockFS::ls(const std::string& path, on_ls_func) const + { + + } + void MockFS::ls(const Dirent& entry, on_ls_func) const + { + + } + List MockFS::ls(const std::string& path) const + { + + } + List MockFS::ls(const Dirent&) const + { + + } + + void MockFS::read(const Dirent&, uint64_t pos, uint64_t n, on_read_func) const + { + + } + Buffer MockFS::read(const Dirent&, uint64_t pos, uint64_t n) const + { + + } + void MockFS::stat(Path_ptr, on_stat_func fn, const Dirent* const ent) const + { + + } + Dirent MockFS::stat(Path, const Dirent* const ent) const + { + + } + void MockFS::cstat(const std::string& pathstr, on_stat_func) + { + + } + void MockFS::init(uint64_t lba, uint64_t size, on_init_func on_init) + { + + } + +} diff --git a/test/lest_util/mock_fs.hpp b/test/lest_util/mock_fs.hpp new file mode 100644 index 0000000000..d8da86d067 --- /dev/null +++ b/test/lest_util/mock_fs.hpp @@ -0,0 +1,34 @@ +#pragma once +#include + +namespace fs +{ + class MockFS : public File_system + { + int device_id() const noexcept override { + return 0; + } + + void ls(const std::string& path, on_ls_func) const override; + void ls(const Dirent& entry, on_ls_func) const override; + List ls(const std::string& path) const override; + List ls(const Dirent&) const override; + + void read(const Dirent&, uint64_t pos, uint64_t n, on_read_func) const override; + Buffer read(const Dirent&, uint64_t pos, uint64_t n) const override; + + void stat(Path_ptr, on_stat_func fn, const Dirent* const = nullptr) const override; + Dirent stat(Path, const Dirent* const = nullptr) const override; + void cstat(const std::string& pathstr, on_stat_func) override; + + std::string name() const override { + return "Mock FS"; + } + + uint64_t block_size() const override { + return 512; + } + + void init(uint64_t lba, uint64_t size, on_init_func on_init) override; + }; +} diff --git a/test/memdisk.fat b/test/memdisk.fat new file mode 100644 index 0000000000000000000000000000000000000000..fc38fe5336988cfb9cf2a71c476a0e02f51deebe GIT binary patch literal 14336 zcmeHt3DhiCRqlW9O$ezZ5=k%#<0Xj-G)YZ86@yZ9Rd?6aT|-w1n3}qVuI}pY8oS_0 z2#8ER1_LtBAoxHK8G;B12|R`X%AhzwfG`LmVNg)O|J8qQ5_o`*wS2Cn?@6uI_4PS> zpHpYtXYYOX!NCD9k`s-RW(UWQ9XmLF&_3TjcJw~=h^OB7J$mzNHz_>7WjtVT6-7=_ ze$opJ)0Gbf>AFe5Aj-qS0dKDd^lN*yN&E~{C$nG8{9WnqKyMe7{=Z)9srI$xJDg^@ z`d{b#36#gFSKTGHRw>zn`g`fkgYKXYsT57BR{_O^OH_kf3-%KG4~-@mOrxA7gE`u^>yf8!tR zcmItAe_HIPBkp$<{z31Lq{}J^^zJK8<`oU5APSa0!`@h)x`_p^!$IjnRoc}2CugvTGN5}V!gNu*G z^857vVk7(iMt9XP%1$oYNjmAK>)ejk*~#3mbKq3#kKOIW1CO@VWo_yWuQ@z?Wc&Yy z*Bu^Gt!(&aiw|nkm;e6ZVc6o$Ene5+8LvM){JR#v`KH6emmQ_we0X?y%YQ?Q4{h<< zKRP_Tw#Apd{qXR{79abL!^4ACR^0r@hllrR@tZ9^sKxI44-fMe^A@jd@$wrF4{vDE z`|#o6cUwH;BZr6GGY$^aj~yN+E#@s=-Qu+^Ufbf$A8*^!;tiiVJiMhv_0xxkXS4!^ z`dnLoi{9tk_Oy7$7Y+{}-{Q3`zPiQyi-(6d9p(RR+pbp2eA(Z%-)ZskZ?*kw@rHjs zJbYe@-R~S8-rVAax6bpiW_oa}KL6O=&OPgnPd#?l`A56y?(Oe&uRT2M+{*Lgc@GY5 z<9$QR+YZU8=koRq9sOQ?czAx>--DkypZS@KM|V1>y5rRczkJsR-VeR}(j&&Bvh()G zz47p{ZT6|>C@*P$m$!3t$?2?9Z8`dTtdS?DT6)%rn@&FD;EJP%pjlo#Nzzl5XLu4< z=~KCY!1K&pB}p7I0=(i0Jm5#TFiu%|ggcu;6mWDr!88=fH`e?|x%t5jU*ih}A^LWB1NRe(g5M~tJq_*5QxdtOX1fS_a*dHTyNU5yA` zrB$KrE)HuniC1F{!E$Q`s~*chmP&^LF+5ERDsyJ^Ai$cB3Tv(eI+83cB!S#Lp?KUf zP`T-{9Uqd9aghj?+i)4NSOpQdLNhonk$5n~kLUmyjN1f_XU*KD8g!RP3$?pyOKe7~ z(4cWRovr4hq$5%7APc>;(I?HkbV|G)rDe?afYo$GjU)$ZrW>kX$C7L4vz;{A?so%o z%!Je!5eK0mtj6P5wT-0Tg$g4L48coR3cOYjpzKf>p4PpStk#` zif$KTGHZgxuHaZ(feX6Pq$-#z?0hG=wUShEXPtCfzw>vs1W+ z*%gF}rV_Rbo{c11Vwt{9Pq1RnJF^Cu+Z;jpax++$vNbpNxtU@pH^e;wTrX3lC#zBo z_HA$`RY5+B1q)~UY8tXU2y^|WTmd~bWl&Ta5S=x(^m@)r0!=Z~3rg&XCRK?AQ6bU5 z)5ogebZMA(OBLtQ-PnXF%7gb9pl|l3HB-uLZPcBk zyX}s4HkXW{WxGXa9JfV9pt}{o9c`p3BoTz!E^;pHL`fG`NW6?Da-^;IPQ9*%B}Lx4 zi9EhiOo6cR+jxN=%Zyh?JIo5Dr*gLto#drQjEp$(a_sZE~Ydc zf%$fnt>8V4fy-JD5|W)67>F##jm<~i{)VeX2){>vok=Zl4VzPDfEle-Et*|OdUFqe0V5-fB+NQRO zolT2z9$5(^54l*Orvp$KcYG!39{5A4Y8NjN5L~i~w$3s|eYs3#mRg}fJ;*l0=y|J8>@w zy?W**+0cl!;cg@Wd zw<<7%t2pFlra4*VOy3-idyp7J3@#M&ZI2ynu~~xiv%G_)Y#QUh5E3T(vWeEiMlA4> zWykB$04alD%y0Sa#uK%@!EoJu5lVTw6FYNzH7sNYwd--LFMS!Pk$j)id^QS3CRhi7 zsKMcIwfEB@J(^U(+^2QL5TuAkVjehkoYEroqetwi!}&i_&4R8u^I_06C+HWQIDYK- z@v{%!^~#HhhpeCbo6vc8IQxE&;~w{|^Nt-m@5}>;hR=b{zQYOaj5}O#9M|9rpr1WT z-0@ss*82CZnB{iv!}o^nb(9I5dojPx{oUHnPa2Wedi}DKa=t$~$vx>;@9niKYQ#e9{TKG8}?pw@S*U7({DWdAHRCy&f(kPw>7oT~EkYXIR*aJZ{Pb3fAFvuzw^A$e&g>S@nOG2-uT1^ z|MjB|@(+LE>5s)8jQ;-NE8fBV-N`rJ=SJ_?ABvA>F8dz)0t=UsRA{*&IU-M9DLn*?t6%|E`ZdA}=PUp@3~`tzRUUk<*$ zc+Q)Umly}i(@Spo`YrD|wV>~D*%=o@KX>(yXE^t$`~_#6 z37t50?C=NIT)S!7HFxab(Qg}E`j+c{=^=lvabG(2uCKlCh0p&L^q!x8(ks5u|G;%$ zSMUGq2mH-->?|(-rGyvXI$5}p7*BLP>*`@xgWaa8RT;tn*Q$edFCI!>9PF>ItxX*rT&wj zz31UWUi$*#S6_443ts)av%fyO>h1>@UD%xXhwnbohE9Iq@dtN!`is_oe)iw&KLbra z_PzT*;u)9t%$?u=4~yr#q5IrtJX@qba)rh`{FcYwS-$?WH@?z%_!sYg$A9@P@&wlPX_j~TsfAf>S^iNa&L4(KK^D8ft@elOACtP=_ z`l1K_@)!T@r<{DvedT|-@W$86&-}Ajy!0~eP3PPboEQ_cj@mwtT%b)g};XU;RUaG z$shdM!(R1A*Zt$SmUn)s&OYUTE_ZJr?*9wN=Z6=c>d8`78E8+4*I=-62Zq#J*oWy#5%zdd=y+OLk>V!O8~UdJt}P`d!ghaeGGl|MpLuF5QwA>27uM$p3dm zwElr|w-;(jw-@@qYS*nT`M$sKr!I~o7sn~p5+Mhr>)~aAyF?ItWw5YLshK+Jq&Z*aG6FumPpL1qj%~i5hcQ1^&_QL zwGCYM#}+&kgjH+C(SbK?Rn@t_kk`amm8lHa89i{-s7=ckqN&*zR!UYtG~?1) z2n{qX5KTH~^Hp!g_SCN1=_iv^+?T<4GY0{U;Toee>LHX4=MFOL!KLc&6BDlxY`Tb5 zXP{RsHJKwp4wm?O)mlddo*14I^5~!_8DK*Nlc|UhwPnj#w-kkKHDm>;+IxK&mN+`~ z2t8R&6Azy0Aw(2oOcBbMULh}|HuK*pXC}RxAVc*Ku+zc%oBX$tQl|celHLqI9zMg8s z&0?lyM!>)(H$ob9FXadI57kVnK9NEDCMwge@NlIVW1|pw?)fg zCOQT+hZq#m<3OdC7?i+^MTL$4jn6?Q-#}^{_tkpQRHU^l$vRt#a}?_{GG?M)gs!B~ zu!_JvK5!{iY`mJdn^sy!-`YTjPX-N>IVlFl%+pkTQeP2+K>Pk6Dl+}b6jph$OSk5Dk!WDyKI zU1#SYQz~lkz_jCu0|SB{XLemuB00)~aFy4SW~&;#mE0L*F}OBzrK33!j;x`GA>$cQ zvc6e@>n^*RIMKYU02|rrkq`-I%R0t&@3v!9St80qDi(|!3474 zbb;G$@!H|nKtxE`N~~#VSXV_vI|`h&Ub;f!6{xojGDeljC1y#TcmYmUViR|Byys+l zL2B*7faP-l>+O)DpSB{!AI|iF75gA$Lb7UF%6_r6L2=7e>s(|Qg+@j=vd5hMs_VKz zQi5EvDf)ocK_O`!c#P8{i#kK1G(!4xFvHd9fY?!7Z#UX5^VlEHm4b%%RXg3XhN{+R zI)xzvasgyIcT{h#;L)aCm=Wvjcq2FbdWX>C0Io%?2a+g**5d?3sJ;WrU1YLD>X8p` zMjK`(0r-gWX#wstsyLf%JQUo|t#;|Qo=2(#w;lp#wzws>qqwM11I=o#^%`V>7Fu&i zkfde>)NnQ=+{q+#ypA3BDHXR!bl#&@_E^9cx{?i`*j+7$f}z={gnJz(=;M5C2plf0 zk%$K{s4OX8iZtCBjNK8o6lc&*qB=tsl9c6xpy-RDvjap|IWkX0QW+V1dVT zf*ZEyl_PET1~H33yZ|>Hu|RAw9ePp~>2r7IfZL8s(AI^8E!YHfESnM& zNon|=uvDYHxGX}4Qw=T;N~Lh3ENwfnMU5Q8;t44h;hZz)tERF|x@v-@45Ubslh$ay zZe50fSPiUcw9?N|h-^2Dun)LTWv}d8k*0aIdpOsd`0^x8k^^wsy z6HIh4CSeF0SH>#J3V&_+1hRB1*x5*xX3cDO=O$*sip6}1Pp5kYgfxf-LK~ZGBAVY1 zr{L)kCW9RnZI(^E$LzXT4mC*Y+IwrqWWL|SOg$cN15VkY?XDuz5J3B2?yO2RYr1*s zYw9Gyl2|9n%v=gvmDpCI$NKA4;*FZ!s1vu!23~lJjbxRzttwNk2PmoJNr>F>glgaM zcBab71wve?0NBi!Qp<;_pY)rs zqgQGfds@nc@VKw8jbNTi7%D(6)2xj}FrTm56Kr0mOghHk+;djI#HW3CS&@yWJIS=2 zbeTYqeU%0t;m$xqUc__Ep2FZ1gDEt%vH^$>`Z6tg+#Jj--v^8?JjPW}TQ2P7O5E3- zZDP_57U=yHT*4yS$twlcoYOLbw6L|yf*GIAkzA`RNeF=j6-{TW-C`;GRD?yN!iTBV zb~~8EBn@@?nAk}gxz}oR6W-sAxBDKt8Zg7L=mv|(Twpf9QX-}K5C`p02_!VBhJ(3+ z=ybNw*k)PG5MhS11AaZ|vE7tXQl2Bn+-w9hsz>ZlzXzCLF<1GdD%d;;x^P(bu>rVe z8V6CqY6c6Eyefi><;Pan?=c&xb57-Z3Gn~fH-+-}SDI+hS1qaiYA^GztXNm}5dy8fg5l!=Xdqd$)6oZ+F-ELc8%;`o)?_q*N#^O%x znXc0VmzwQ+(9|qzwZF3$n$^oJyWBK$!)`Cj0Jvzix8BfL)GY%U;Pv9zoUEKf7Skjj_kk957>W==vA`p7g`UKBfxHNhnFjx zF%`-P$CkUAce|pEd-T;@S) z(8A6{3&z}==~fgs_Iwx5dUiyZH4<5r;ZY6&WxI(K_LGQ-7fVZb1hKS%pu`4+Fyw;>OE;hMA(ZE#H5r$e?UWkn* z`g+7F`F6-R7}0_9Fc`<3QMUHY9n>9XN{8PJ>1MNS{4F6e#=3E%6c|Rtl&Q5PMrR7) z%qI1CRr{TkTBpr=UBUEPP2-?F4@=smatY&hxLlbFdyqSEuUQfsfNGVjMXUc>1ww)N zlvhc8A`zXU>sl!k^KscSz(D!OL#y|io2H=0r+JvYoJqL;d~*ahXnA7Y6YkKrgr-Bz*?P}A{eA+^MW0S9h& z7(XCMcr4V*Hj<#dB{5@ATkNEclSsjQ;SMsc?@CiN$@u{mAV6(&1#}4qGLCdP%AVw~ zwh^r|P_=N633%*hn^rt4D$j;ugwurDD&2b#@fIL9P5Y(vRcsP`L!<~fj z_AXfEsjuy3KWP6cWU@C_MHe?T9i4Pb-jf+A+5*GIT+8mFXT+w@dnjkls%@(ctSXY% zQe&8H$RgTqa|a`uK{lQ3deQ`0Gb?J*+5(HxnF10x=eAy}t84SPL3T^nmc?yYPC&G4 zu!U}vh}=!{CU9gB>g1f;ox{G;P=O{SD3Wb7V8M}1Z!eM?P?12z=chFvg{hMdm${HE zH{&se&u0-y^&_-a1(6&h0XfPjglsQ4;dEsIK`mo*QnF%QnCv(eDI>%{g`Q$qg&ObJ z$;`(s*tU|QM+^wqmupxV?)dRAhUosh5hx%oUE6}t%o$AD!D);L9~bjizXGSYUX&G${=4x0TWsul)~5kNr`qY5X{k?0X#8)P8Ed*2U;U~AEqwP?5@ zH?jI&s4Ft-jJDeuzZ$n4X!NkLyKF{=0Jl9lU)4`(i)TvX)H`RdE06ln^09fa5LJqU2R5xKcQ;|C+(9L={)^aH(tX#^Kk>;6b z=#wf+x?6$Qi?zB}pvBB$oPN~KQ3kZ8bex3|3G)O=ws*)zz3EfXh%FKX2O5^)0mMN@_?m>{e6SAQPnxCdvEabka|U z8$N8^bCzDT>XTkWSqiQD2D>U9!v@yf^{(#QxIdR*G{hW36Lm3J$jTZE1WxWoGRpJp zY9PvxJORZG)5UFlEn5D(H>+5N1tx{#tTcsi`*45l%HAj{Es`o|e6$havY{gyDE9e!IhoIJ#PqwJj?n@2N@-M0ZOy!J(wP#8(Ah|> z`7r5JUNaRr*LKyJUl(p*P=a6pdapOzRGJ>eL#Ax+od@;`atNwBn8rg$>$M>PG!xNk z)pyBlyprN=sB>XoCuNT}(f!D4{Q#*?4XUDCbx9)am|iG05-lMjmzP7+WIEo0O`4#~ zw=TEVY-YFBtpjYzLAiAcU@t9~JX(19EECYU6hW})YgjcSVV{P_so@UmZe(Ml9ZbMx zcjZYP-sZiaUm|r6Z(2`;9rxtjT1Y#4bX_`YZ$-DeNA^R~X=+VqI_yF;`pp7H@ex>txPIn!Evbi+vK(bJE1BqaK50b=O@(^_Y<8;b1Y(hyWw~t3 z{y@%Mw$YPWQpuZ<4s^QQveV?oj;zIPo}gi@Ne&UOi*d}>c_$Q+h*|`x&SHaoD7Qz+ zknFcgw`yDMJO>B~tZl9C2~zDQNxu(g1HI>{X?I@Hg<;IcanD7nrPPOTu3W~ZJ6QQ> z)Q^(@Gn2g>NKpsqFCZN$P<=TcB*?8JoR-Wo3Z^DAl4nj1Z*?TEd69RNSrC{F$%INe z!Yaax$Xx|s<3#V|*=hq8t(wlaW*{n63P|wPx;>NxOzn=L)kNQkF0bWPL`PXH2upM} zN$7klqZ(l2i^U2Z#r>hdLZxm_H!~YHd%H5q%YDENN@(h+<~|Til^QV2*jRC$Hb9NB zcCMBUP#>7KvU>;**v3U-s4phISC2%2_Vko>4Jl8uHj>H7U~q4hV=yRUnrqJmCq~<0 zQuo8%vUP1TTWwgix}J~72xGRj+iehn98t=uU(K40Woc^- zo=!u@9Rd#Opd209-Q5CIj2SqT25g-U6Fwa67H_fru4eKePGB#5r{*FeJv2A~nU`F)2!};;fKnQH7$%*wALSUQGR! z3NR8z&*UB~hd30sI^Z;F)*WlgYpfLeo1SO(%>4wKu8>+uleiu@9hie#ZGwco{nGE4 zKv(J2`-JGB*laeO4rp@Mx|1Q;j?583=(RcU=YBP=+h$Omx>nULX(SzL85Dd@)UAvH z_|atNkOd+tRUsK^PxVL4sK|*;H(J7QpQspO*KvA9K(r>{m;(osHY{WsvkQ;wTW$Oo zun0TJDHjGYwileGVx}c)%aS8{R)*-U;fu=UyaXAcJ5-3qVDoKzt*{=Ca~>Q8bQ>!v zz``P#jg`rkoRSsj=3KLo`F^8M2X-5)7Whdr^5>u=k33EsN)gX;={#S~xTQh2S~Cia z3SR680Z)SFpq%GZEpP7ul+BVbw~&-r-FXLDNv4=|rD>4heJLERRk<`+?{N&Khw-%(uhzi+(q{{rxmlY#&M literal 0 HcmV?d00001 From ea2c96cd00f6aa07f904ec5a5d08073839480459 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 8 Jun 2018 11:39:05 +0200 Subject: [PATCH 403/723] test: Try to use INCLUDEOS_ROOT to find unittest file --- test/fs/unit/unit_fat.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/fs/unit/unit_fat.cpp b/test/fs/unit/unit_fat.cpp index 1002db50ea..2a5fd23f78 100644 --- a/test/fs/unit/unit_fat.cpp +++ b/test/fs/unit/unit_fat.cpp @@ -9,7 +9,13 @@ static Disk_ptr disk = nullptr; CASE("Prepare custom memdisk") { - auto* fp = fopen("../memdisk.fat", "rb"); + const char* rootp(getenv("INCLUDEOS_SRC")); + std::string path; + if (rootp == nullptr) path = ".."; + else path = std::string(rootp) + "/test"; + + path += "/memdisk.fat"; + auto* fp = fopen(path.c_str(), "rb"); EXPECT(fp != nullptr); fseek(fp, 0L, SEEK_END); long int size = ftell(fp); From a069c0a2168cf8f6bf10b9b0fc31ca128393496e Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 8 Jun 2018 11:51:58 +0200 Subject: [PATCH 404/723] vmrunner: Unique device id for each drive --- vmrunner/vmrunner.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 897b2eb6e6..a2479d597e 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -321,6 +321,7 @@ def __init__(self, config): self._stopped = False self._sudo = False self._image_name = self._config if "image" in self._config else self.name() + " vm" + self.m_drive_no = 0 # Pretty printing self.info = Logger(color.INFO("<" + type(self).__name__ + ">")) @@ -347,9 +348,11 @@ def drive_arg(self, filename, device = "virtio", drive_format = "raw", media_typ if device in names: device = names[device] + driveno = "drv" + str(self.m_drive_no) + self.m_drive_no += 1 return ["-drive", "file=" + filename + ",format=" + drive_format - + ",if=none" + ",media=" + media_type + ",id=drv0", - "-device", device + ",drive=drv0,serial=foo"] + + ",if=none" + ",media=" + media_type + ",id=" + driveno, + "-device", device + ",drive=" + driveno +",serial=foo"] # -initrd "file1 arg=foo,file2" # This syntax is only available with multiboot. From c7deceead308d44232f29328081a0bca1e0d1e9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 8 Jun 2018 15:03:24 +0200 Subject: [PATCH 405/723] net: Moved IANA defines to own header to avoid big include --- api/net/iana.hpp | 85 +++++++++++++++++++++++++++++++++++++++++ api/net/inet_common.hpp | 78 +------------------------------------ 2 files changed, 86 insertions(+), 77 deletions(-) create mode 100644 api/net/iana.hpp diff --git a/api/net/iana.hpp b/api/net/iana.hpp new file mode 100644 index 0000000000..e45f44a61c --- /dev/null +++ b/api/net/iana.hpp @@ -0,0 +1,85 @@ + +#pragma once + +#include + +namespace net { + + /* RFC 6335 - IANA */ + namespace port_ranges + { + static constexpr uint16_t SYSTEM_START {0}; + static constexpr uint16_t SYSTEM_END {1023}; + static constexpr uint16_t USER_START {1024}; + static constexpr uint16_t USER_END {49151}; + static constexpr uint16_t DYNAMIC_START {49152}; + static constexpr uint16_t DYNAMIC_END {65535}; + + static constexpr bool is_dynamic(const uint16_t port) noexcept + { return port >= DYNAMIC_START; } + } + + /** + * Known transport layer protocols. Shared between IPv4 and IPv6. + * Full list: + * http://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml + */ + enum class Protocol : uint8_t { + HOPOPT = 0, + ICMPv4 = 1, + IPv4 = 4, // IPv4 encapsulation + TCP = 6, + UDP = 17, + IPv6 = 41, // IPv6 encapsulation + ICMPv6 = 58, + IPv6_NONXT = 59, + OPTSV6 = 60 + }; + + /** + * Explicit Congestion Notification (ECN) values for IPv4 and IPv6 + * Defined in RFC3168 + **/ + enum class ECN : uint8_t { + NOT_ECT = 0b00, // Non-ECN transport + ECT_0 = 0b01, // ECN-enabled transport 1 + ECT_1 = 0b10, // ECN-enabled transport 2 + CE = 0b11 // Congestion encountered + }; + + /** + * Differentiated Services Code Points, for IPv4 and IPv6 + * Defined in RFC2474 + * NOTE: Replaces IPv4 TOS field + * + * IANA list: + * https://www.iana.org/assignments/dscp-registry/dscp-registry.xhtml + * + * 6 DSCP bits together with 2 ECN bits form one octet + **/ + enum class DSCP : uint8_t { + CS0 = 0b000000, + CS1 = 0b001000, + CS2 = 0b010000, + CS3 = 0b011000, + CS4 = 0b100000, + CS5 = 0b101000, + CS6 = 0b110000, + CS7 = 0b111000, + AF11 = 0b001010, + AF12 = 0b001100, + AF13 = 0b001110, + AF21 = 0b010010, + AF22 = 0b010100, + AF23 = 0b010110, + AF31 = 0b011010, + AF32 = 0b011100, + AF33 = 0b011110, + AF41 = 0b100010, + AF42 = 0b100100, + AF43 = 0b100110, + EF_PHB = 0b101110, + VOICE_ADMIT = 0b101100 + }; + +} diff --git a/api/net/inet_common.hpp b/api/net/inet_common.hpp index 01b89a87fd..d75cefffdc 100644 --- a/api/net/inet_common.hpp +++ b/api/net/inet_common.hpp @@ -30,6 +30,7 @@ #include #include +#include namespace net { // Packet must be forward declared to avoid circular dependency @@ -64,83 +65,6 @@ namespace net { return std::unique_ptr(d); } - /* RFC 6335 - IANA */ - namespace port_ranges - { - static constexpr uint16_t SYSTEM_START {0}; - static constexpr uint16_t SYSTEM_END {1023}; - static constexpr uint16_t USER_START {1024}; - static constexpr uint16_t USER_END {49151}; - static constexpr uint16_t DYNAMIC_START {49152}; - static constexpr uint16_t DYNAMIC_END {65535}; - - static constexpr bool is_dynamic(const uint16_t port) noexcept - { return port >= DYNAMIC_START; } - } - - /** - * Known transport layer protocols. Shared between IPv4 and IPv6. - * Full list: - * http://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml - */ - enum class Protocol : uint8_t { - HOPOPT = 0, - ICMPv4 = 1, - IPv4 = 4, // IPv4 encapsulation - TCP = 6, - UDP = 17, - IPv6 = 41, // IPv6 encapsulation - ICMPv6 = 58, - IPv6_NONXT = 59, - OPTSV6 = 60 - }; - - /** - * Explicit Congestion Notification (ECN) values for IPv4 and IPv6 - * Defined in RFC3168 - **/ - enum class ECN : uint8_t { - NOT_ECT = 0b00, // Non-ECN transport - ECT_0 = 0b01, // ECN-enabled transport 1 - ECT_1 = 0b10, // ECN-enabled transport 2 - CE = 0b11 // Congestion encountered - }; - - /** - * Differentiated Services Code Points, for IPv4 and IPv6 - * Defined in RFC2474 - * NOTE: Replaces IPv4 TOS field - * - * IANA list: - * https://www.iana.org/assignments/dscp-registry/dscp-registry.xhtml - * - * 6 DSCP bits together with 2 ECN bits form one octet - **/ - enum class DSCP : uint8_t { - CS0 = 0b000000, - CS1 = 0b001000, - CS2 = 0b010000, - CS3 = 0b011000, - CS4 = 0b100000, - CS5 = 0b101000, - CS6 = 0b110000, - CS7 = 0b111000, - AF11 = 0b001010, - AF12 = 0b001100, - AF13 = 0b001110, - AF21 = 0b010010, - AF22 = 0b010100, - AF23 = 0b010110, - AF31 = 0b011010, - AF32 = 0b011100, - AF33 = 0b011110, - AF41 = 0b100010, - AF42 = 0b100100, - AF43 = 0b100110, - EF_PHB = 0b101110, - VOICE_ADMIT = 0b101100 - }; - inline uint16_t new_ephemeral_port() noexcept { return port_ranges::DYNAMIC_START + rand() % (port_ranges::DYNAMIC_END - port_ranges::DYNAMIC_START); } From 25335f8a41c856dffea02151752f20ec9678f8e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 8 Jun 2018 15:09:50 +0200 Subject: [PATCH 406/723] tcp: Transmit IP6 works, removed some callbacks, started to remove existence of tcp Packet which is replaced by view --- api/net/tcp/common.hpp | 53 +++++++++++++++++++++---- api/net/tcp/connection.hpp | 48 +++-------------------- api/net/tcp/connection.inc | 10 ----- api/net/tcp/packet.hpp | 5 --- api/net/tcp/packet4_view.hpp | 15 ++++--- api/net/tcp/packet6_view.hpp | 13 ++++++- api/net/tcp/packet_view.hpp | 65 ++++++++++++++++++++++++++++--- api/net/tcp/tcp.hpp | 13 ++++--- src/net/inet.cpp | 18 ++++++--- src/net/tcp/connection.cpp | 28 +++---------- src/net/tcp/connection_states.cpp | 2 - src/net/tcp/tcp.cpp | 15 ++++--- 12 files changed, 169 insertions(+), 116 deletions(-) diff --git a/api/net/tcp/common.hpp b/api/net/tcp/common.hpp index 4f2494b80b..220b538567 100644 --- a/api/net/tcp/common.hpp +++ b/api/net/tcp/common.hpp @@ -20,9 +20,9 @@ #define NET_TCP_COMMON_HPP #include -#include #include #include +#include namespace net { namespace tcp { @@ -65,15 +65,13 @@ namespace net { return std::make_shared> (std::forward (args)...); } - class Packet; - using Packet_ptr = std::unique_ptr; - class Packet_view; using Packet_view_ptr = std::unique_ptr; class Connection; using Connection_ptr = std::shared_ptr; + // TODO: Remove when TCP packet class is gone template uint16_t calculate_checksum(const Tcp_packet& packet) { @@ -93,12 +91,53 @@ namespace net { return net::checksum(sum, buffer, length); } + template + uint16_t calculate_checksum4(const View4& packet) + { + constexpr uint8_t Proto_TCP = 6; // avoid including inet_common + uint16_t length = packet.tcp_length(); + const auto ip_src = packet.ip4_src(); + const auto ip_dst = packet.ip4_dst(); + // Compute sum of pseudo-header + uint32_t sum = + (ip_src.whole >> 16) + + (ip_src.whole & 0xffff) + + (ip_dst.whole >> 16) + + (ip_dst.whole & 0xffff) + + (Proto_TCP << 8) + + htons(length); + + // Compute sum of header and data + const char* buffer = (char*) &packet.tcp_header(); + return net::checksum(sum, buffer, length); + } + template uint16_t calculate_checksum6(const View6& packet) { - // Alf fix me ;D - (void) packet; - return 0; + constexpr uint8_t Proto_TCP = 6; // avoid including inet_common + uint16_t length = packet.tcp_length(); + const auto ip_src = packet.ip6_src(); + const auto ip_dst = packet.ip6_dst(); + // Compute sum of pseudo-header + uint32_t sum = 0; + + for(int i = 0; i < 4; i++) + { + uint32_t part = ip_src.template get_part(i); + sum += (part >> 16); + sum += (part & 0xffff); + + part = ip_dst.template get_part(i); + sum += (part >> 16); + sum += (part & 0xffff); + } + + sum += (Proto_TCP << 8) + htons(length); + + // Compute sum of header and data + const char* buffer = (char*) &packet.tcp_header(); + return net::checksum(sum, buffer, length); } } // < namespace tcp diff --git a/api/net/tcp/connection.hpp b/api/net/tcp/connection.hpp index e7106fa174..d4b6038b8e 100644 --- a/api/net/tcp/connection.hpp +++ b/api/net/tcp/connection.hpp @@ -20,7 +20,6 @@ #define NET_TCP_CONNECTION_HPP #include "common.hpp" -#include "packet.hpp" #include "packet_view.hpp" #include "read_request.hpp" #include "rttm.hpp" @@ -139,27 +138,6 @@ class Connection { /** Called with the packet that got dropped and the reason why. */ using PacketDroppedCallback = delegate; - /** - * @brief Event when a connection has dropped a packet. - * Useful for debugging/track counting. - * - * @param[in] callback The callback - * - * @return This connection - */ - inline Connection& on_packet_dropped(PacketDroppedCallback callback); - - /** Called with the number of simultaneous retransmit attempts and the current Round trip timeout in milliseconds. */ - using RtxTimeoutCallback = delegate; - /** - * @brief Event when the connections retransmit timer has expired. - * Useful for debugging/track counting. - * - * @param[in] callback The callback - * - * @return This connection - */ - inline Connection& on_rtx_timeout(RtxTimeoutCallback); /** * @brief Only change the on_read callback without touching the buffer. @@ -651,8 +629,6 @@ class Connection { /** Callbacks */ ConnectCallback on_connect_; DisconnectCallback on_disconnect_; - PacketDroppedCallback on_packet_dropped_; - RtxTimeoutCallback on_rtx_timeout_; CloseCallback on_close_; /** Retransmission timer */ @@ -814,11 +790,8 @@ class Connection { void signal_disconnect(Disconnect::Reason&& reason) { on_disconnect_(retrieve_shared(), Disconnect{reason}); } - void signal_packet_dropped(const Packet& packet, Drop_reason reason) - { if(on_packet_dropped_) on_packet_dropped_(packet, reason); } - void signal_rtx_timeout() - { if(on_rtx_timeout_) on_rtx_timeout_(rtx_attempt_+1, rttm.rto_ms()); } + { } /* Drop a packet. Used for debug/callback. @@ -965,20 +938,20 @@ class Connection { * * @return The amount of data filled into the packet. */ - size_t fill_packet(Packet& packet, const uint8_t* data, size_t n) + size_t fill_packet(Packet_view& packet, const uint8_t* data, size_t n) { return packet.fill(data, std::min(n, (size_t)SMSS())); } /* Transmit the packet and hooks up retransmission. */ - void transmit(Packet_ptr); + void transmit(Packet_view_ptr); /* Creates a new outgoing packet with the current TCB values and options. */ - Packet_ptr create_outgoing_packet(); + Packet_view_ptr create_outgoing_packet(); - Packet_ptr outgoing_packet() + Packet_view_ptr outgoing_packet() { return create_outgoing_packet(); } /** @@ -1160,17 +1133,8 @@ class Connection { /* Add an option. */ - void add_option(Option::Kind, Packet&); + void add_option(Option::Kind, Packet_view&); - /** - * @brief Parses the timestamp option from a packet (if any). - * Assumes the packet contains no other options. - * - * @param[in] A TCP packet - * - * @return A pointer the the timestamp option (nullptr if none) - */ - Option::opt_ts* parse_ts_option(const Packet&) const; }; // < class Connection diff --git a/api/net/tcp/connection.inc b/api/net/tcp/connection.inc index 6f796007c4..166d3ef0e2 100644 --- a/api/net/tcp/connection.inc +++ b/api/net/tcp/connection.inc @@ -24,16 +24,6 @@ inline Connection& Connection::on_write(WriteCallback cb) { return *this; } -inline Connection& Connection::on_packet_dropped(PacketDroppedCallback callback) { - on_packet_dropped_ = callback; - return *this; -} - -inline Connection& Connection::on_rtx_timeout(RtxTimeoutCallback cb) { - on_rtx_timeout_ = cb; - return *this; -} - inline Connection& Connection::on_close(CloseCallback cb) { on_close_ = cb; return *this; diff --git a/api/net/tcp/packet.hpp b/api/net/tcp/packet.hpp index dc80751ad4..62ed38837a 100644 --- a/api/net/tcp/packet.hpp +++ b/api/net/tcp/packet.hpp @@ -26,11 +26,6 @@ #include "common.hpp" // constants, seq_t #include "headers.hpp" -inline unsigned round_up(unsigned n, unsigned div) { - Expects(div > 0); - return (n + div - 1) / div; -} - namespace net { namespace tcp { diff --git a/api/net/tcp/packet4_view.hpp b/api/net/tcp/packet4_view.hpp index c309e1cfd8..db23431c98 100644 --- a/api/net/tcp/packet4_view.hpp +++ b/api/net/tcp/packet4_view.hpp @@ -18,11 +18,17 @@ class Packet4_view : public Packet_view { inline void init(); uint16_t compute_tcp_checksum() const noexcept override - { return calculate_checksum6(*this); } + { return calculate_checksum4(*this); } Protocol ipv() const noexcept override { return Protocol::IPv4; } + ip4::Addr ip4_src() const noexcept + { return packet().ip_src(); } + + ip4::Addr ip4_dst() const noexcept + { return packet().ip_dst(); } + private: PacketIP4& packet() noexcept { return static_cast(*pkt); } @@ -47,16 +53,13 @@ class Packet4_view : public Packet_view { uint16_t ip_header_length() const noexcept override { return packet().ip_header_length(); } + }; inline void Packet4_view::init() { - //memset(header, 0, sizeof(tcp::Header)); - auto* ipdata = packet().ip_data().data(); - // clear TCP header - ((uint32_t*) ipdata)[3] = 0; - ((uint32_t*) ipdata)[4] = 0; + memset(header, 0, sizeof(tcp::Header)); set_win(tcp::default_window_size); header->offset_flags.offset_reserved = (5 << 4); diff --git a/api/net/tcp/packet6_view.hpp b/api/net/tcp/packet6_view.hpp index 3d1eeda051..cff5288fa4 100644 --- a/api/net/tcp/packet6_view.hpp +++ b/api/net/tcp/packet6_view.hpp @@ -21,7 +21,13 @@ class Packet6_view : public Packet_view { { return calculate_checksum6(*this); } Protocol ipv() const noexcept override - { return Protocol::IPv4; } + { return Protocol::IPv6; } + + ip6::Addr ip6_src() const noexcept + { return packet().ip_src(); } + + ip6::Addr ip6_dst() const noexcept + { return packet().ip_dst(); } private: PacketIP6& packet() noexcept @@ -52,6 +58,11 @@ class Packet6_view : public Packet_view { inline void Packet6_view::init() { + // clear TCP header + memset(header, 0, sizeof(tcp::Header)); + + set_win(tcp::default_window_size); + header->offset_flags.offset_reserved = (5 << 4); set_length(); } diff --git a/api/net/tcp/packet_view.hpp b/api/net/tcp/packet_view.hpp index 0a14f25eae..54f2b5beeb 100644 --- a/api/net/tcp/packet_view.hpp +++ b/api/net/tcp/packet_view.hpp @@ -2,16 +2,18 @@ #pragma once #include "common.hpp" +#include "headers.hpp" #include "options.hpp" +#include +#include +#include + namespace net::tcp { class Packet_view { public: - Header& tcp_header() noexcept - { return *header; } - const Header& tcp_header() const noexcept { return *header; } @@ -98,6 +100,9 @@ class Packet_view { auto tcp_header_length() const noexcept { return offset() * 4; } + uint16_t tcp_length() const + { return tcp_header_length() + tcp_data_length(); } + virtual uint16_t compute_tcp_checksum() const noexcept = 0; Packet_view& set_tcp_checksum(uint16_t checksum) noexcept @@ -146,7 +151,6 @@ class Packet_view { template void add_tcp_option_aligned(Args&&... args); - const Option::opt_ts* ts_option() const noexcept { return ts_opt; } @@ -167,7 +171,23 @@ class Packet_view { bool has_tcp_data() const noexcept { return tcp_data_length() > 0; } - // View operations + inline size_t fill(const uint8_t* buffer, size_t length); + + // Util // + + seq_t end() const noexcept + { return seq() + tcp_data_length(); } + + bool is_acked_by(const seq_t ack) const noexcept + { return ack >= (seq() + tcp_data_length()); } + + bool should_rtx() const noexcept + { return has_tcp_data() or isset(SYN) or isset(FIN); } + + inline std::string to_string() const; + + // Packet_view specific operations // + net::Packet_ptr release() { Expects(pkt != nullptr && "Packet ptr is already null"); @@ -193,6 +213,9 @@ class Packet_view { Expects(pkt != nullptr); } + Header& tcp_header() noexcept + { return *header; } + void set_header(uint8_t* hdr) { Expects(hdr != nullptr); header = reinterpret_cast(hdr); } @@ -200,6 +223,7 @@ class Packet_view { void set_length(uint16_t newlen = 0) { pkt->set_data_end(ip_header_length() + tcp_header_length() + newlen); } + private: Option::opt_ts* ts_opt = nullptr; @@ -211,8 +235,15 @@ class Packet_view { // TODO: see if we can get rid of these virtual calls virtual uint16_t ip_data_length() const noexcept = 0; virtual uint16_t ip_header_length() const noexcept = 0; + uint16_t ip_capacity() const noexcept + { return pkt->capacity() - ip_header_length(); } + }; +inline unsigned round_up(unsigned n, unsigned div) { + Expects(div > 0); + return (n + div - 1) / div; +} template inline void Packet_view::add_tcp_option(Args&&... args) { @@ -280,4 +311,28 @@ inline const Option::opt_ts* Packet_view::parse_ts_option() noexcept return this->ts_opt; } +inline size_t Packet_view::fill(const uint8_t* buffer, size_t length) +{ + size_t rem = ip_capacity() - tcp_length(); + if(rem == 0) return 0; + size_t total = std::min(length, rem); + // copy from buffer to packet buffer + memcpy(tcp_data() + tcp_data_length(), buffer, total); + // set new packet length + set_length(tcp_data_length() + total); + return total; +} + +inline std::string Packet_view::to_string() const +{ + char buffer[512]; + int len = snprintf(buffer, sizeof(buffer), + "[ S:%s D:%s SEQ:%u ACK:%u HEAD-LEN:%d OPT-LEN:%d DATA-LEN:%d" + " WIN:%u FLAGS:%#x ]", + source().to_string().c_str(), destination().to_string().c_str(), + seq(), ack(), tcp_header_length(), tcp_options_length(), + tcp_data_length(), win(), tcp_header().offset_flags.flags); + return std::string(buffer, len); +} + } diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index 7cc7c57fb2..3e4304745c 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -23,7 +23,6 @@ #include "connection.hpp" #include "headers.hpp" #include "listener.hpp" -#include "packet.hpp" #include "packet_view.hpp" #include // connections, listeners @@ -48,7 +47,7 @@ namespace net { using CleanupCallback = tcp::Connection::CleanupCallback; using ConnectCallback = tcp::Connection::ConnectCallback; - using Packet_reroute_func = delegate; + using Packet_reroute_func = delegate; using Port_utils = std::map; @@ -196,7 +195,10 @@ namespace net { * @param[in] del A downstream delegate */ void set_network_out(downstream del) - { _network_layer_out = del; } + { network_layer_out_ = del; } + + void set_network_out6(downstream del) + { network_layer_out6_ = del; } /** * @brief Returns a collection of the listeners for this instance. @@ -491,7 +493,8 @@ namespace net { Port_utils& ports_; - downstream _network_layer_out; + downstream network_layer_out_; + downstream network_layer_out6_; /** Internal writeq - connections gets queued in the wait for packets and recvs offer */ std::deque writeq; @@ -533,7 +536,7 @@ namespace net { * * @param[in] A TCP Segment */ - void transmit(tcp::Packet_ptr); + void transmit(tcp::Packet_view_ptr); /** * @brief Creates an outgoing TCP packet. diff --git a/src/net/inet.cpp b/src/net/inet.cpp index 0e36e7cd17..9b3b924158 100644 --- a/src/net/inet.cpp +++ b/src/net/inet.cpp @@ -19,6 +19,7 @@ #include #include #include +#include // due to ICMP error //temp using namespace net; @@ -98,6 +99,9 @@ Inet::Inet(hw::Nic& nic) // TCP -> IP4 tcp_.set_network_out(ip4_top); + // TCP -> IP6 + tcp_.set_network_out6(ip6_top); + // IP4 -> Arp ip4_.set_linklayer_out(arp_top); @@ -130,20 +134,22 @@ void Inet::error_report(Error& err, Packet_ptr orig_pckt) { bool too_big = false; // Get the destination to the original packet - Socket dest = [](const PacketIP4& pkt)->Socket { - switch (pkt.ip_protocol()) { + Socket dest = [](std::unique_ptr& pkt)->Socket { + switch (pkt->ip_protocol()) { case Protocol::UDP: { - const auto& udp = static_cast(pkt); + const auto& udp = static_cast(*pkt); return udp.destination(); } case Protocol::TCP: { - const auto& tcp = static_cast(pkt); - return tcp.destination(); + auto tcp = tcp::Packet4_view(std::move(pkt)); + auto dst = tcp.destination(); + pkt = static_unique_ptr_cast(tcp.release()); + return dst; } default: return {}; } - }(*pckt_ip4); + }(pckt_ip4); if (err.is_icmp()) { diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index f59fe4e9c4..d74a412ca9 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -18,10 +18,9 @@ // #define DEBUG // #define DEBUG2 -#include // Ensures/Expects +#include // Ensures/Expects #include #include -#include #include #include @@ -113,8 +112,6 @@ void Connection::reset_callbacks() on_disconnect_ = {this, &Connection::default_on_disconnect}; on_connect_.reset(); writeq.on_write(nullptr); - on_packet_dropped_.reset(); - on_rtx_timeout_.reset(); on_close_.reset(); if(read_request) @@ -316,7 +313,7 @@ void Connection::deserialize_from(void*) {} __attribute__((weak)) int Connection::serialize_to(void*) const { return 0; } -Packet_ptr Connection::create_outgoing_packet() +Packet_view_ptr Connection::create_outgoing_packet() { auto packet = (is_ipv6_) ? host_.create_outgoing_packet6() : host_.create_outgoing_packet(); @@ -347,10 +344,10 @@ Packet_ptr Connection::create_outgoing_packet() packet->set_seq(cb.SND.NXT).set_ack(cb.RCV.NXT); debug(" Outgoing packet created: %s \n", packet->to_string().c_str()); - return static_unique_ptr_cast(packet->release()); + return packet; } -void Connection::transmit(Packet_ptr packet) { +void Connection::transmit(Packet_view_ptr packet) { if(!cb.SND.TS_OK and !rttm.active() and packet->end() == cb.SND.NXT) @@ -1095,8 +1092,6 @@ void Connection::clean_up() { on_connect_.reset(); on_disconnect_.reset(); - on_packet_dropped_.reset(); - on_rtx_timeout_.reset(); on_close_.reset(); if(read_request) read_request->callback.reset(); @@ -1219,7 +1214,7 @@ void Connection::parse_options(const Packet_view& packet) { } } -void Connection::add_option(Option::Kind kind, Packet& packet) { +void Connection::add_option(Option::Kind kind, Packet_view& packet) { switch(kind) { @@ -1250,16 +1245,6 @@ void Connection::add_option(Option::Kind kind, Packet& packet) { } } -Option::opt_ts* Connection::parse_ts_option(const Packet& packet) const -{ - auto* opt = packet.tcp_options(); - - while(((Option*)opt)->kind == Option::NOP and opt < (uint8_t*)packet.tcp_data()) - opt++; - - return (((Option*)opt)->kind == Option::TS) ? (Option::opt_ts*)opt : nullptr; -} - bool Connection::uses_window_scaling() const noexcept { return host_.uses_wscale(); @@ -1275,9 +1260,8 @@ bool Connection::uses_SACK() const noexcept return host_.uses_SACK(); } -void Connection::drop(const Packet_view& packet, Drop_reason reason) +void Connection::drop(const Packet_view& packet, [[maybe_unused]]Drop_reason reason) { - signal_packet_dropped(reinterpret_cast(*packet.packet_ptr()), reason); host_.drop(packet); } diff --git a/src/net/tcp/connection_states.cpp b/src/net/tcp/connection_states.cpp index fbf0802334..be854997e5 100644 --- a/src/net/tcp/connection_states.cpp +++ b/src/net/tcp/connection_states.cpp @@ -15,9 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include // Ensures/Expects #include -#include using namespace net::tcp; using namespace std; diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index 5af371ab55..104bbe1ab2 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -224,7 +224,7 @@ void TCP::receive(Packet_view& packet) // Redirect packet to custom function if (packet_rerouter) { - packet_rerouter(static_unique_ptr_cast(packet.release())); + packet_rerouter(packet.release()); return; } @@ -359,16 +359,21 @@ void TCP::reset_pmtu(Socket dest, IP4::PMTU pmtu) { } } -void TCP::transmit(tcp::Packet_ptr packet) { +void TCP::transmit(tcp::Packet_view_ptr packet) +{ // Generate checksum. packet->set_tcp_checksum(); - debug2(" %s\n", packet->to_string().c_str()); // Stat increment bytes transmitted and packets transmitted (*bytes_tx_) += packet->tcp_data_length(); (*packets_tx_)++; - _network_layer_out(std::move(packet)); + if(packet->ipv() == Protocol::IPv6) { + network_layer_out6_(packet->release()); + } + else { + network_layer_out_(packet->release()); + } } tcp::Packet_view_ptr TCP::create_outgoing_packet() @@ -397,7 +402,7 @@ void TCP::send_reset(const tcp::Packet_view& in) out->set_source(in.destination()); out->set_destination(in.source()); - transmit(static_unique_ptr_cast(out->release())); + transmit(std::move(out)); } seq_t TCP::generate_iss() { From 276dd04a5568a66233586d8ad73f3300d06bd4d0 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Sun, 10 Jun 2018 22:47:57 +0200 Subject: [PATCH 407/723] examples: Add some test scripts to TLS server, and add vanilla HTTP support --- examples/TLS_server/art.yaml | 15 +++++++++++++++ examples/TLS_server/fuzz.py | 26 ++++++++++++++++++++++++++ examples/TLS_server/service.cpp | 14 ++++++++++---- examples/TLS_server/vm.json | 3 +++ 4 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 examples/TLS_server/art.yaml create mode 100755 examples/TLS_server/fuzz.py create mode 100644 examples/TLS_server/vm.json diff --git a/examples/TLS_server/art.yaml b/examples/TLS_server/art.yaml new file mode 100644 index 0000000000..a3c2cbae6a --- /dev/null +++ b/examples/TLS_server/art.yaml @@ -0,0 +1,15 @@ +config: + target: "http://10.0.0.42" + phases: + - duration: 60 + arrivalRate: 10 + plugins: + fuzzer: {} +scenarios: + - name: "Fuzz some stuff" + flow: + - get: + url: "/?hello-{{naughtyString}}" + - log: "***** naughtyString = {{ naughtyString }}" + - get: + url: "/something/else?{{naughtyString}}" diff --git a/examples/TLS_server/fuzz.py b/examples/TLS_server/fuzz.py new file mode 100755 index 0000000000..2d052e5707 --- /dev/null +++ b/examples/TLS_server/fuzz.py @@ -0,0 +1,26 @@ +#!/usr/bin/python3 +import socket #for sockets +import sys #for exit + +remote_ip = '10.0.0.42' +port = 80 + +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.connect((remote_ip , port)) + +payload="t/" * 16000000 + +#Send some data to remote server +message = str.encode("GET / HTTP/1.1\r\nContent-Length: 16000000\r\n\r\n%s" % payload) + +#print("SENDING >>> " + str(message)) +try : + #Set the whole string + s.sendall(message) +except socket.error: + #Send failed + print('Send failed') + sys.exit() + +data = s.recv(512) +print(data) diff --git a/examples/TLS_server/service.cpp b/examples/TLS_server/service.cpp index e901c2ed0d..25f34e915a 100644 --- a/examples/TLS_server/service.cpp +++ b/examples/TLS_server/service.cpp @@ -21,7 +21,7 @@ #include #include #define BENCHMARK_MODE -static const bool ENABLE_TLS = true; +static const bool ENABLE_TLS = false; static const bool USE_BOTAN_TLS = false; static http::Server* server = nullptr; @@ -54,11 +54,14 @@ void Service::start() "blabla", ca_key, ca_cert, srv_key, inet.tcp()); printf("Using Botan for HTTPS transport\n"); } - else { + else if (ENABLE_TLS) { server = new http::OpenSSL_server( "/test.pem", "/test.key", inet.tcp()); printf("Using OpenSSL for HTTPS transport\n"); } + else { + server = new http::Server(inet.tcp()); + } server->on_request( [] (auto request, auto response_writer) { @@ -66,8 +69,11 @@ void Service::start() response_writer->write(); }); +if (ENABLE_TLS) // listen on default HTTPS port server->listen(443); +else + server->listen(80); } #ifdef BENCHMARK_MODE @@ -76,14 +82,14 @@ static void print_heap_info() { const std::string heapinfo = HeapDiag::to_string(); printf("%s\n", heapinfo.c_str()); - StackSampler::print(10); + //StackSampler::print(10); } void Service::ready() { using namespace std::chrono; Timers::periodic(1s, [] (int) { - print_heap_info(); + //print_heap_info(); }); StackSampler::begin(); diff --git a/examples/TLS_server/vm.json b/examples/TLS_server/vm.json new file mode 100644 index 0000000000..86f982bd24 --- /dev/null +++ b/examples/TLS_server/vm.json @@ -0,0 +1,3 @@ +{ + "mem": 10 +} From 90439d98f949be19b35496b6a6d2ece7a74718b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 11 Jun 2018 12:29:14 +0200 Subject: [PATCH 408/723] ip6: Don't move array because of gcc reasons --- api/net/ip6/addr.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index 7894b0c720..82cb8dcc2e 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -62,7 +62,7 @@ struct Addr { : i64{a.i64} {} Addr(Addr&& a) noexcept - : i64{std::move(a.i64)} {} + : i64{a.i64} {} // returns this IPv6 Address as a string std::string str() const { @@ -178,7 +178,7 @@ struct Addr { Addr& operator=(Addr&& other) noexcept { - i64 = std::move(other.i64); + i64 = other.i64; return *this; } From 4fefb72d45e707882da1615166dbfe09ace9d132 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 11 Jun 2018 12:30:28 +0200 Subject: [PATCH 409/723] tcp: Improvements to ip6 handling --- api/net/tcp/listener.hpp | 2 +- api/net/tcp/packet.hpp | 5 +++++ api/net/tcp/packet_view.hpp | 2 +- api/net/tcp/tcp.hpp | 30 +++++++------------------- src/net/tcp/tcp.cpp | 43 +++++++++++++++++++++++++++++-------- 5 files changed, 49 insertions(+), 33 deletions(-) diff --git a/api/net/tcp/listener.hpp b/api/net/tcp/listener.hpp index 230bab90b6..6f0af4df74 100644 --- a/api/net/tcp/listener.hpp +++ b/api/net/tcp/listener.hpp @@ -64,7 +64,7 @@ class Listener { * * @return The local Socket the listener is bound to */ - Socket local() const noexcept + const Socket& local() const noexcept { return local_; } port_t port() const noexcept diff --git a/api/net/tcp/packet.hpp b/api/net/tcp/packet.hpp index 62ed38837a..dc80751ad4 100644 --- a/api/net/tcp/packet.hpp +++ b/api/net/tcp/packet.hpp @@ -26,6 +26,11 @@ #include "common.hpp" // constants, seq_t #include "headers.hpp" +inline unsigned round_up(unsigned n, unsigned div) { + Expects(div > 0); + return (n + div - 1) / div; +} + namespace net { namespace tcp { diff --git a/api/net/tcp/packet_view.hpp b/api/net/tcp/packet_view.hpp index 54f2b5beeb..6efbf9fff6 100644 --- a/api/net/tcp/packet_view.hpp +++ b/api/net/tcp/packet_view.hpp @@ -325,7 +325,7 @@ inline size_t Packet_view::fill(const uint8_t* buffer, size_t length) inline std::string Packet_view::to_string() const { - char buffer[512]; + char buffer[256]; int len = snprintf(buffer, sizeof(buffer), "[ S:%s D:%s SEQ:%u ACK:%u HEAD-LEN:%d OPT-LEN:%d DATA-LEN:%d" " WIN:%u FLAGS:%#x ]", diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index 3e4304745c..e7695fe0d9 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -91,7 +91,7 @@ namespace net { * * @return A TCP Listener */ - tcp::Listener& listen(Socket socket, ConnectCallback cb = nullptr); + tcp::Listener& listen(const Socket& socket, ConnectCallback cb = nullptr); /** * @brief Close a Listener @@ -101,7 +101,7 @@ namespace net { * @param socket listening socket * @return whether the listener existed and was closed */ - bool close(Socket socket); + bool close(const Socket& socket); /** * @brief Make an outgoing connection to a TCP remote (IP:port). @@ -400,7 +400,7 @@ namespace net { * * @return True if bound, False otherwise. */ - bool is_bound(const Socket socket) const; + bool is_bound(const Socket& socket) const; /** * @brief Number of connections queued for writing. @@ -413,7 +413,7 @@ namespace net { /** * @brief The IP address for which the TCP instance is "connected". * - * @return An IP4 address + * @return An IP address */ tcp::Address address() const noexcept; @@ -598,7 +598,7 @@ namespace net { * * @param[in] socket The socket */ - void bind(const Socket socket); + void bind(const Socket& socket); /** * @brief Unbinds a socket, making it free for future use @@ -607,7 +607,7 @@ namespace net { * * @return Returns wether there was a socket that got unbound */ - bool unbind(const Socket socket); + bool unbind(const Socket& socket); /** * @brief Bind to an socket where the address is given and the @@ -618,16 +618,7 @@ namespace net { * * @return The socket that got bound. */ - Socket bind(const tcp::Address addr); - - /** - * @brief Binds to an socket where the address is given by the - * stack and port is ephemeral. See bind(addr) - * - * @return The socket that got bound. - */ - Socket bind() - { return bind(address()); } + Socket bind(const tcp::Address& addr); /** * @brief Determines if the source address is valid. @@ -694,12 +685,7 @@ namespace net { * * @param[in] listener A Listener */ - void close_listener(tcp::Listener& listener) - { - const auto socket = listener.local(); - unbind(socket); - listeners_.erase(socket); - } + void close_listener(tcp::Listener& listener); // WRITEQ HANDLING diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index 104bbe1ab2..e0f8fddf64 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -87,7 +87,7 @@ void TCP::smp_process_writeq(size_t packets) SMP::signal(this->cpu_id); } -Listener& TCP::listen(Socket socket, ConnectCallback cb) +Listener& TCP::listen(const Socket& socket, ConnectCallback cb) { bind(socket); @@ -116,8 +116,10 @@ Listener& TCP::listen(const tcp::port_t port, ConnectCallback cb, const bool ipv return *listener; } -bool TCP::close(Socket socket) { - //TODO: check ip6 any addr +bool TCP::close(const Socket& socket) +{ + // TODO: if the socket is ipv6 any addr it will also + // close the ipv4 any addr due to call to Listener::close() auto it = listeners_.find(socket); if(it != listeners_.end()) { @@ -126,12 +128,16 @@ bool TCP::close(Socket socket) { Ensures(listeners_.find(socket) == listeners_.end()); return true; } + return false; } void TCP::connect(Socket remote, ConnectCallback callback) { - create_connection(bind(), remote, std::move(callback))->open(true); + auto addr = remote.address().is_v6() + ? Addr{inet_.ip6_addr()} : Addr{inet_.ip_addr()}; + + create_connection(bind(addr), remote, std::move(callback))->open(true); } void TCP::connect(Address source, Socket remote, ConnectCallback callback) @@ -147,7 +153,10 @@ void TCP::connect(Socket local, Socket remote, ConnectCallback callback) Connection_ptr TCP::connect(Socket remote) { - auto conn = create_connection(bind(), remote); + auto addr = remote.address().is_v6() + ? Addr{inet_.ip6_addr()} : Addr{inet_.ip_addr()}; + + auto conn = create_connection(bind(addr), remote); conn->open(true); return conn; } @@ -421,7 +430,7 @@ void TCP::drop(const tcp::Packet_view&) { debug(" Packet dropped\n"); } -bool TCP::is_bound(const Socket socket) const +bool TCP::is_bound(const Socket& socket) const { auto it = ports_.find(socket.address()); @@ -434,7 +443,7 @@ bool TCP::is_bound(const Socket socket) const return false; } -void TCP::bind(const Socket socket) +void TCP::bind(const Socket& socket) { if(UNLIKELY( is_valid_source(socket.address()) == false )) throw TCP_error{"Cannot bind to address: " + socket.address().to_string()}; @@ -445,7 +454,7 @@ void TCP::bind(const Socket socket) ports_[socket.address()].bind(socket.port()); } -Socket TCP::bind(const Address addr) +Socket TCP::bind(const Address& addr) { if(UNLIKELY( is_valid_source(addr) == false )) throw TCP_error{"Cannot bind to address: " + addr.to_string()}; @@ -457,7 +466,7 @@ Socket TCP::bind(const Address addr) return {addr, port}; } -bool TCP::unbind(const Socket socket) +bool TCP::unbind(const Socket& socket) { auto it = ports_.find(socket.address()); @@ -553,6 +562,22 @@ bool TCP::is_valid_source(const tcp::Address& addr) const noexcept void TCP::kick() { process_writeq(inet_.transmit_queue_available()); } +void TCP::close_listener(tcp::Listener& listener) +{ + const auto& socket = listener.local(); + unbind(socket); + listeners_.erase(socket); + + // if the listener is "dual-stack", make sure to clean up the + // ip4 any addr copy as well + if(socket.address().is_v6() and socket.address().is_any()) + { + Socket ip4_sock{ip4::Addr::addr_any, socket.port()}; + unbind(ip4_sock); + listeners_.erase(ip4_sock); + } +} + TCP::Listeners::iterator TCP::find_listener(const Socket& socket) { Listeners::iterator it = listeners_.find(socket); From 97b2c69b631e4388f9742e3c284793ea40983070 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 11 Jun 2018 12:31:17 +0200 Subject: [PATCH 410/723] test: Dont reference or validate _end in unittests --- lib/LiveUpdate/update.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index 3de76b1fa4..cb34bcdef4 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -115,9 +115,11 @@ void LiveUpdate::exec(const buffer_t& blob, void* location) if (storage_area < (char*) 0x200) { throw std::runtime_error("LiveUpdate storage area is (probably) a null pointer"); } +#ifndef PLATFORM_UNITTEST if (storage_area >= &_ELF_START_ && storage_area < &_end) { throw std::runtime_error("LiveUpdate storage area is inside kernel area"); } +#endif if (storage_area >= (char*) OS::heap_begin() && storage_area < (char*) OS::heap_end()) { throw std::runtime_error("LiveUpdate storage area is inside the heap area"); } From 8936e23d4f641baa90d8304d06f6b025ca0edebe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 11 Jun 2018 12:32:08 +0200 Subject: [PATCH 411/723] test: Explicit ip4 Socket construction due to redesign of socket --- test/lest_util/packet_factory.hpp | 4 ++-- test/net/unit/addr_test.cpp | 1 + test/net/unit/conntrack_test.cpp | 20 ++++++++++---------- test/net/unit/napt_test.cpp | 16 ++++++++-------- test/net/unit/nat_test.cpp | 10 +++++----- test/net/unit/path_mtu_discovery.cpp | 10 +++++----- test/net/unit/router_test.cpp | 4 ++-- test/net/unit/socket.cpp | 14 +++++++------- test/net/unit/tcp_packet_test.cpp | 8 ++++---- 9 files changed, 44 insertions(+), 43 deletions(-) diff --git a/test/lest_util/packet_factory.hpp b/test/lest_util/packet_factory.hpp index 4743994ac6..64c955a76d 100644 --- a/test/lest_util/packet_factory.hpp +++ b/test/lest_util/packet_factory.hpp @@ -101,8 +101,8 @@ static std::unique_ptr create_udp_packet_init(Socket src, Socket ip4->init(Protocol::UDP); auto udp = net::static_unique_ptr_cast (std::move(ip4)); udp->init(src.port(), dst.port()); - udp->set_ip_src(src.address()); - udp->set_ip_dst(dst.address()); + udp->set_ip_src(src.address().v4()); + udp->set_ip_dst(dst.address().v4()); return udp; } diff --git a/test/net/unit/addr_test.cpp b/test/net/unit/addr_test.cpp index e9c144a9cc..ae0872e340 100644 --- a/test/net/unit/addr_test.cpp +++ b/test/net/unit/addr_test.cpp @@ -27,6 +27,7 @@ CASE("Addr") ip4::Addr ip4addr; ip6::Addr ip6addr; + EXPECT(addr.is_any()); //EXPECT(addr.v4() == ip4addr); EXPECT(addr.v6() == ip6addr); diff --git a/test/net/unit/conntrack_test.cpp b/test/net/unit/conntrack_test.cpp index 0b574036e8..2814720ffb 100644 --- a/test/net/unit/conntrack_test.cpp +++ b/test/net/unit/conntrack_test.cpp @@ -22,8 +22,8 @@ CASE("Testing Conntrack flow") { using namespace net; const Protocol proto{Protocol::UDP}; - Socket src{{10,0,0,42}, 80}; - Socket dst{{10,0,0,1}, 1337}; + Socket src{ip4::Addr{10,0,0,42}, 80}; + Socket dst{ip4::Addr{10,0,0,1}, 1337}; Quadruple quad{src, dst}; // Reversed quadruple Quadruple rquad = quad; rquad.swap(); @@ -90,8 +90,8 @@ CASE("Testing Conntrack update entry") { using namespace net; const Protocol proto{Protocol::UDP}; - Socket src{{10,0,0,42}, 80}; - Socket dst{{10,0,0,1}, 1337}; + Socket src{ip4::Addr{10,0,0,42}, 80}; + Socket dst{ip4::Addr{10,0,0,1}, 1337}; Quadruple quad{src, dst}; // Reversed quadruple Quadruple rquad = quad; rquad.swap(); @@ -111,8 +111,8 @@ CASE("Testing Conntrack update entry") EXPECT(entry->second == rquad); // Let's update the second quadruple with a new one - Socket new_src{{10,0,0,42}, 80}; - Socket new_dst{{10,0,0,1}, 1337}; + Socket new_src{ip4::Addr{10,0,0,42}, 80}; + Socket new_dst{ip4::Addr{10,0,0,1}, 1337}; Quadruple new_quad{new_src, new_dst}; ct.update_entry(proto, entry->second, new_quad); @@ -130,8 +130,8 @@ CASE("Testing Conntrack update entry") CASE("Testing Conntrack limit") { using namespace net; - Socket src{{10,0,0,42}, 80}; - Socket dst{{10,0,0,1}, 1337}; + Socket src{ip4::Addr{10,0,0,42}, 80}; + Socket dst{ip4::Addr{10,0,0,1}, 1337}; Quadruple quad{src, dst}; // Reversed quadruple Quadruple rquad = quad; rquad.swap(); @@ -163,8 +163,8 @@ CASE("Testing Conntrack limit") CASE("Testing Conntrack serialization") { using namespace net; - Socket src{{10,0,0,42}, 80}; - Socket dst{{10,0,0,1}, 1337}; + Socket src{ip4::Addr{10,0,0,42}, 80}; + Socket dst{ip4::Addr{10,0,0,1}, 1337}; Quadruple quad{src, dst}; // Reversed quadruple Quadruple rquad = quad; rquad.swap(); diff --git a/test/net/unit/napt_test.cpp b/test/net/unit/napt_test.cpp index 3e258b007e..b985731b9c 100644 --- a/test/net/unit/napt_test.cpp +++ b/test/net/unit/napt_test.cpp @@ -31,7 +31,7 @@ static Conntrack::Entry* get_entry(Conntrack& ct, const PacketIP4& pkt) return entry; } -static tcp::Packet_ptr tcp_packet(Socket src, Socket dst) +static std::unique_ptr tcp_packet(Socket src, Socket dst) { auto tcp = create_tcp_packet_init(src, dst); tcp->set_tcp_checksum(); @@ -51,9 +51,9 @@ CASE("NAPT DNAT") auto conntrack = std::make_shared(); NAPT napt{conntrack}; - const Socket src{{10,0,0,1}, 32222}; - const Socket dst{{10,0,0,42},80}; - const Socket target{{10,0,0,43}, 8080}; + const Socket src{ip4::Addr{10,0,0,1}, 32222}; + const Socket dst{ip4::Addr{10,0,0,42},80}; + const Socket target{ip4::Addr{10,0,0,43}, 8080}; // TCP // Request @@ -104,8 +104,8 @@ CASE("NAPT SNAT") auto conntrack = std::make_shared(); NAPT napt{conntrack}; - const Socket src{{10,0,0,1}, 32222}; - const Socket dst{{10,0,0,42},80}; + const Socket src{ip4::Addr{10,0,0,1}, 32222}; + const Socket dst{ip4::Addr{10,0,0,42},80}; const ip4::Addr new_src{10,0,0,10}; // TCP @@ -164,8 +164,8 @@ CASE("NAPT MASQUERADE") Inet inet{nic}; inet.network_config({10,0,0,40},{255,255,255,0}, 0); - const Socket src{{10,0,0,1}, 32222}; - const Socket dst{{10,0,0,42},80}; + const Socket src{ip4::Addr{10,0,0,1}, 32222}; + const Socket dst{ip4::Addr{10,0,0,42},80}; // TCP // Request diff --git a/test/net/unit/nat_test.cpp b/test/net/unit/nat_test.cpp index 531cee2eab..11796471d8 100644 --- a/test/net/unit/nat_test.cpp +++ b/test/net/unit/nat_test.cpp @@ -25,8 +25,8 @@ using namespace net::nat; CASE("TCP NAT verifying rewrite and checksum") { // Socket - const Socket src{{10,0,0,42},80}; - const Socket dst{{10,0,0,43},32222}; + const Socket src{ip4::Addr{10,0,0,42},80}; + const Socket dst{ip4::Addr{10,0,0,43},32222}; auto tcp = create_tcp_packet_init(src, dst); EXPECT(tcp->source() == src); EXPECT(tcp->destination() == dst); @@ -107,8 +107,8 @@ CASE("TCP NAT verifying rewrite and checksum") CASE("UDP NAT verifying rewrite") { // Socket - const Socket src{{10,0,0,42},80}; - const Socket dst{{10,0,0,43},32222}; + const Socket src{ip4::Addr{10,0,0,42},80}; + const Socket dst{ip4::Addr{10,0,0,43},32222}; auto udp = create_udp_packet_init(src, dst); EXPECT(udp->source() == src); EXPECT(udp->destination() == dst); @@ -212,7 +212,7 @@ CASE("ICMP NAT verifying rewrite") EXPECT(ip4.ip_dst() == src); EXPECT(ip4.compute_ip_checksum() == 0); - const Socket sock{{10,10,10,10},80}; + const Socket sock{ip4::Addr{10,10,10,10},80}; // Socket does nothing (unsupported) dnat(ip4, sock); EXPECT(ip4.ip_src() == dst); diff --git a/test/net/unit/path_mtu_discovery.cpp b/test/net/unit/path_mtu_discovery.cpp index b29d1f0f1b..d3946f9423 100644 --- a/test/net/unit/path_mtu_discovery.cpp +++ b/test/net/unit/path_mtu_discovery.cpp @@ -98,7 +98,7 @@ CASE("Turning Path MTU Discovery off clears the PMTU cache") ICMP_error err{icmp4::Type::DEST_UNREACHABLE, (uint8_t) icmp4::code::Dest_unreachable::FRAGMENTATION_NEEDED, 1400}; bool too_big = err.is_too_big(); - Socket dest{{10,0,0,48}, 443}; + Socket dest{ip4::Addr{10,0,0,48}, 443}; inet.ip_obj().update_path(dest, err.pmtu(), too_big); @@ -116,7 +116,7 @@ CASE("No PMTU entry exists for non-existing path") inet.set_path_mtu_discovery(true); - EXPECT(inet.ip_obj().pmtu(Socket{{10,0,0,45}, 80}) == 0); + EXPECT(inet.ip_obj().pmtu(Socket{ip4::Addr{10,0,0,45}, 80}) == 0); } CASE("A PMTU entry can be removed") @@ -128,7 +128,7 @@ CASE("A PMTU entry can be removed") ICMP_error err{icmp4::Type::DEST_UNREACHABLE, (uint8_t) icmp4::code::Dest_unreachable::FRAGMENTATION_NEEDED, 1400}; bool too_big = err.is_too_big(); - Socket dest{{10,0,0,49}, 443}; + Socket dest{ip4::Addr{10,0,0,49}, 443}; inet.ip_obj().update_path(dest, err.pmtu(), too_big); @@ -148,11 +148,11 @@ CASE("The PMTU cache/map can be flushed") ICMP_error err1{icmp4::Type::DEST_UNREACHABLE, (uint8_t) icmp4::code::Dest_unreachable::FRAGMENTATION_NEEDED, 1000}; bool too_big1 = err1.is_too_big(); - Socket dest1{{10,0,0,5}, 80}; + Socket dest1{ip4::Addr{10,0,0,5}, 80}; ICMP_error err2{icmp4::Type::DEST_UNREACHABLE, (uint8_t) icmp4::code::Dest_unreachable::FRAGMENTATION_NEEDED, 900}; bool too_big2 = err2.is_too_big(); - Socket dest2{{10,0,0,6}, 80}; + Socket dest2{ip4::Addr{10,0,0,6}, 80}; inet.ip_obj().update_path(dest1, err1.pmtu(), too_big1); inet.ip_obj().update_path(dest2, err2.pmtu(), too_big2); diff --git a/test/net/unit/router_test.cpp b/test/net/unit/router_test.cpp index 4fa78ee5ee..4d64656351 100644 --- a/test/net/unit/router_test.cpp +++ b/test/net/unit/router_test.cpp @@ -176,8 +176,8 @@ CASE("net::router: Actual routing verifying TTL") inet1.set_forward_delg(router.forward_delg()); inet2.set_forward_delg(router.forward_delg()); - const Socket src{{10,0,1,10}, 32222}; - const Socket dst{{10,0,2,10}, 80}; + const Socket src{ip4::Addr{10,0,1,10}, 32222}; + const Socket dst{ip4::Addr{10,0,2,10}, 80}; const uint8_t DEFAULT_TTL = PacketIP4::DEFAULT_TTL; // Here we gonna receive the ICMP TTL Exceeded ONCE diff --git a/test/net/unit/socket.cpp b/test/net/unit/socket.cpp index 99c6e1ccc5..0362c2c8af 100644 --- a/test/net/unit/socket.cpp +++ b/test/net/unit/socket.cpp @@ -24,10 +24,10 @@ using namespace net; CASE("Creating a Socket without arguments is empty") { Socket socket; - const Socket::Address addr { 0,0,0,0 }; + const Socket::Address addr { ip4::Addr{0,0,0,0} }; EXPECT( socket.is_empty() ); EXPECT( socket.address() == addr ); - EXPECT( socket.address() == 0 ); + EXPECT( socket.address().is_any() ); EXPECT( socket.port() == 0 ); const std::string expected_str {"0.0.0.0:0"}; @@ -36,7 +36,7 @@ CASE("Creating a Socket without arguments is empty") CASE("Creating a Socket with arguments is not empty") { - const Socket::Address addr { 10,0,0,42 }; + const Socket::Address addr { ip4::Addr{10,0,0,42} }; const Socket::port_t port = 80; Socket socket { addr, 80 }; @@ -51,15 +51,15 @@ CASE("Creating a Socket with arguments is not empty") CASE("Sockets can be compared to each other") { - Socket sock1 { { 10,0,0,42 }, 80 }; - Socket sock2 { { 10,0,0,42 }, 8080 }; - Socket sock3 { { 192,168,0,1 }, 80 }; + Socket sock1 { ip4::Addr{ 10,0,0,42 }, 80 }; + Socket sock2 { ip4::Addr{ 10,0,0,42 }, 8080 }; + Socket sock3 { ip4::Addr{ 192,168,0,1 }, 80 }; EXPECT_NOT( sock1 == sock2 ); EXPECT_NOT( sock1 == sock3 ); EXPECT( sock2 != sock3 ); - const Socket temp { { 10,0,0,42 }, 80 }; + const Socket temp { ip4::Addr{ 10,0,0,42 }, 80 }; EXPECT( sock1 == temp ); const Socket empty; diff --git a/test/net/unit/tcp_packet_test.cpp b/test/net/unit/tcp_packet_test.cpp index d56a0fd821..fbabbffbb3 100644 --- a/test/net/unit/tcp_packet_test.cpp +++ b/test/net/unit/tcp_packet_test.cpp @@ -174,8 +174,8 @@ CASE("TCP header source and dest") auto tcp = create_tcp_packet(); tcp->init(); - tcp->set_source({{10,0,0,1}, 666}); - tcp->set_destination({{10,0,0,2}, 667}); + tcp->set_source({ip4::Addr{10,0,0,1}, 666}); + tcp->set_destination({ip4::Addr{10,0,0,2}, 667}); EXPECT(tcp->source().address() == ip4::Addr(10,0,0,1)); EXPECT(tcp->source().port() == 666); @@ -188,8 +188,8 @@ CASE("TCP checksum") auto tcp = create_tcp_packet(); tcp->init(); - tcp->set_source({{10,0,0,1}, 666}); - tcp->set_destination({{10,0,0,2}, 667}); + tcp->set_source({ip4::Addr{10,0,0,1}, 666}); + tcp->set_destination({ip4::Addr{10,0,0,2}, 667}); tcp->set_tcp_checksum(); EXPECT(tcp->compute_tcp_checksum() == 0); From 91d35fcafe05e1f369e597a730c8d493f60608bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 11 Jun 2018 12:33:19 +0200 Subject: [PATCH 412/723] test: Build net Addr test --- test/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index abba07a694..5cc116be26 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -97,6 +97,7 @@ set(TEST_SOURCES ${TEST}/kernel/unit/unit_events.cpp ${TEST}/kernel/unit/unit_timers.cpp ${TEST}/kernel/unit/unit_liveupdate.cpp + ${TEST}/net/unit/addr_test.cpp ${TEST}/net/unit/bufstore.cpp ${TEST}/net/unit/checksum.cpp ${TEST}/net/unit/cidr.cpp From 6f241965f973a0346a082f8cdee620fe76e50362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 11 Jun 2018 13:30:42 +0200 Subject: [PATCH 413/723] net: Fixed endianess in Addr when ip4, extended unittest --- api/net/addr.hpp | 10 ++++----- test/net/unit/addr_test.cpp | 45 ++++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/api/net/addr.hpp b/api/net/addr.hpp index 52c10e9751..782c6873ec 100644 --- a/api/net/addr.hpp +++ b/api/net/addr.hpp @@ -28,13 +28,13 @@ union Addr { : ip6_{} {} Addr(ip4::Addr addr) noexcept - : ip4_{0, ip4_signature, std::move(addr)} {} + : ip4_{0, ip4_sign_le, std::move(addr)} {} Addr(ip6::Addr addr) noexcept : ip6_{std::move(addr)} {} bool is_v4() const noexcept - { return ip4_.big == 0 and ip4_.sign == ip4_signature; } + { return ip4_.big == 0 and ip4_.sign == ip4_sign_le; } bool is_v6() const noexcept { return not is_v4(); } @@ -42,7 +42,7 @@ union Addr { void set_v4(ip4::Addr addr) noexcept { ip4_.big = 0; - ip4_.sign = ip4_signature; + ip4_.sign = ip4_sign_le; ip4_.addr = std::move(addr); } @@ -62,7 +62,7 @@ union Addr { { return ip6_; } bool is_any() const noexcept - { return (is_v4() and ip4_.addr == 0) or ip6_ == ip6::Addr::addr_any; } + { return ip6_ == ip6::Addr::addr_any or (is_v4() and ip4_.addr == 0); } Addr any_addr() const noexcept { return is_v4() ? Addr{ip4::Addr::addr_any} : Addr{ip6::Addr::addr_any}; } @@ -116,7 +116,7 @@ union Addr { } ip4_; ip6::Addr ip6_; - static constexpr uint32_t ip4_signature{0x0000FFFF}; + static constexpr uint32_t ip4_sign_le{0xFFFF0000}; }; static_assert(sizeof(Addr) == sizeof(ip6::Addr)); diff --git a/test/net/unit/addr_test.cpp b/test/net/unit/addr_test.cpp index ae0872e340..372eb304ad 100644 --- a/test/net/unit/addr_test.cpp +++ b/test/net/unit/addr_test.cpp @@ -20,19 +20,48 @@ using namespace net; -CASE("Addr") +CASE("Default Addr intitialization") { - Addr addr{}; + Addr addr; - ip4::Addr ip4addr; ip6::Addr ip6addr; - EXPECT(addr.is_any()); - //EXPECT(addr.v4() == ip4addr); + EXPECT(addr.is_v6()); + EXPECT_NOT(addr.is_v4()); EXPECT(addr.v6() == ip6addr); + EXPECT_THROWS(addr.v4()); + + const std::string str{"0:0:0:0:0:0:0:0"}; + EXPECT(addr.to_string() == str); +} + +CASE("Addr v4/v6") +{ + Addr addr{ip4::Addr::addr_any}; + + EXPECT(addr.is_any()); + EXPECT(addr.is_v4()); + EXPECT_NOT(addr.is_v6()); + EXPECT(addr.v4() == ip4::Addr::addr_any); + + EXPECT(addr.to_string() == std::string("0.0.0.0")); + + const ip4::Addr ipv4{10,0,0,42}; + addr.set_v4(ipv4); + EXPECT_NOT(addr.is_any()); + EXPECT(addr.is_v4()); + EXPECT(addr.v4() == ipv4); + + EXPECT(addr.to_string() == std::string("10.0.0.42")); + + const ip6::Addr ipv6{0,0,0x0000FFFF,ntohl(ipv4.whole)}; + EXPECT(addr.v6() == ipv6); + EXPECT(addr.v6().to_string() == std::string("0:0:0:0:0:ffff:a00:2a")); - addr.set_v4({10,0,0,42}); - printf("%s\n", addr.v4().to_string().c_str()); - printf("%s\n", addr.v6().to_string().c_str()); + addr.set_v6(ip6::Addr{0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd}); + EXPECT_NOT(addr.is_v4()); + EXPECT(addr.is_v6()); + EXPECT_NOT(addr.is_any()); + EXPECT(addr.to_string() == std::string("fe80:0:0:0:e823:fcff:fef4:85bd")); } From ddb3c0f3cbfa097da1b4b1565b6402a2c22f8f12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 11 Jun 2018 16:19:44 +0200 Subject: [PATCH 414/723] test: Fixed failing unittests --- api/net/addr.hpp | 8 ++++---- api/net/ip6/addr.hpp | 10 ++++++++-- api/net/socket.hpp | 12 +++++++----- api/util/syslog_facility.hpp | 2 +- test/lest_util/packet_factory.hpp | 1 + test/net/unit/socket.cpp | 4 ++-- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/api/net/addr.hpp b/api/net/addr.hpp index 782c6873ec..3be93f6575 100644 --- a/api/net/addr.hpp +++ b/api/net/addr.hpp @@ -28,13 +28,13 @@ union Addr { : ip6_{} {} Addr(ip4::Addr addr) noexcept - : ip4_{0, ip4_sign_le, std::move(addr)} {} + : ip4_{0, ip4_sign_be, std::move(addr)} {} Addr(ip6::Addr addr) noexcept : ip6_{std::move(addr)} {} bool is_v4() const noexcept - { return ip4_.big == 0 and ip4_.sign == ip4_sign_le; } + { return ip4_.big == 0 and ip4_.sign == ip4_sign_be; } bool is_v6() const noexcept { return not is_v4(); } @@ -42,7 +42,7 @@ union Addr { void set_v4(ip4::Addr addr) noexcept { ip4_.big = 0; - ip4_.sign = ip4_sign_le; + ip4_.sign = ip4_sign_be; ip4_.addr = std::move(addr); } @@ -116,7 +116,7 @@ union Addr { } ip4_; ip6::Addr ip6_; - static constexpr uint32_t ip4_sign_le{0xFFFF0000}; + static constexpr uint32_t ip4_sign_be{0xFFFF0000}; }; static_assert(sizeof(Addr) == sizeof(ip6::Addr)); diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index 82cb8dcc2e..a7e2ee231b 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -186,7 +186,7 @@ struct Addr { * Operator to check for equality */ bool operator==(const Addr& other) const noexcept - { return i32 == other.i32; } + { return i64 == other.i64; } /** * Operator to check for inequality @@ -198,7 +198,13 @@ struct Addr { * Operator to check for greater-than relationship */ bool operator>(const Addr& other) const noexcept - { return i32 > other.i32; } + { + if(ntohl(i32[0]) > ntohl(other.i32[0])) return true; + if(ntohl(i32[1]) > ntohl(other.i32[1])) return true; + if(ntohl(i32[2]) > ntohl(other.i32[2])) return true; + if(ntohl(i32[3]) > ntohl(other.i32[3])) return true; + return false; + } /** * Operator to check for greater-than-or-equal relationship diff --git a/api/net/socket.hpp b/api/net/socket.hpp index 228ff3dd2f..65a36ca256 100644 --- a/api/net/socket.hpp +++ b/api/net/socket.hpp @@ -93,8 +93,10 @@ class Socket { * * @return A string representation of this class */ - std::string to_string() const - { return address().to_string() + ":" + std::to_string(port()); } + std::string to_string() const { + return (addr_.is_v6()) ? "[" + addr_.to_string() + "]:" + std::to_string(port_) + : addr_.to_string() + ":" + std::to_string(port_); + } /** * Check if this socket is empty <0.0.0.0:0> @@ -114,7 +116,7 @@ class Socket { */ bool operator==(const Socket& other) const noexcept { - return addr_ == other.addr_; + return addr_ == other.addr_ and port_ == other.port_; } /** @@ -139,8 +141,8 @@ class Socket { */ bool operator<(const Socket& other) const noexcept { - return (address() < other.address()) - or ((address() == other.address()) and (port() < other.port())); + return (addr_ < other.addr_) + or ((addr_ == other.addr_) and (port_ < other.port_)); } /** diff --git a/api/util/syslog_facility.hpp b/api/util/syslog_facility.hpp index ddf463aeb1..eea518f839 100644 --- a/api/util/syslog_facility.hpp +++ b/api/util/syslog_facility.hpp @@ -187,7 +187,7 @@ class Syslog_print : public Syslog_facility { public: void syslog(const std::string& log_message) override; void settings(const net::UDP::addr_t, const net::UDP::port_t) override {} - net::UDP::addr_t ip() const noexcept override { return net::ip6::Addr::addr_any; } + net::UDP::addr_t ip() const noexcept override { return net::ip4::Addr::addr_any; } net::UDP::port_t port() const noexcept override { return 0; } void open_socket() override {} void close_socket() override {} diff --git a/test/lest_util/packet_factory.hpp b/test/lest_util/packet_factory.hpp index 64c955a76d..e0ae879195 100644 --- a/test/lest_util/packet_factory.hpp +++ b/test/lest_util/packet_factory.hpp @@ -81,6 +81,7 @@ static std::unique_ptr create_ip6_packet_init(ip6::Addr src, ip6 static std::unique_ptr create_tcp_packet() noexcept { auto ip4 = create_ip4_packet(); + ip4->init(Protocol::TCP); auto tcp = net::static_unique_ptr_cast (std::move(ip4)); return tcp; } diff --git a/test/net/unit/socket.cpp b/test/net/unit/socket.cpp index 0362c2c8af..69d7d72e33 100644 --- a/test/net/unit/socket.cpp +++ b/test/net/unit/socket.cpp @@ -24,13 +24,13 @@ using namespace net; CASE("Creating a Socket without arguments is empty") { Socket socket; - const Socket::Address addr { ip4::Addr{0,0,0,0} }; + const Socket::Address addr { ip6::Addr::addr_any }; EXPECT( socket.is_empty() ); EXPECT( socket.address() == addr ); EXPECT( socket.address().is_any() ); EXPECT( socket.port() == 0 ); - const std::string expected_str {"0.0.0.0:0"}; + const std::string expected_str {"[0:0:0:0:0:0:0:0]:0"}; EXPECT( socket.to_string() == expected_str ); } From 5024ef2b31ece9004cd7eddcaed58e86aeb93a10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 12 Jun 2018 12:48:10 +0200 Subject: [PATCH 415/723] net: Simpler construction of Addr --- api/net/addr.hpp | 11 +++++++++++ api/net/inet.hpp | 2 +- api/net/ip6/addr.hpp | 2 +- test/net/unit/ip6_packet_test.cpp | 2 +- test/net/unit/socket.cpp | 10 +++++----- 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/api/net/addr.hpp b/api/net/addr.hpp index 3be93f6575..ad763e0848 100644 --- a/api/net/addr.hpp +++ b/api/net/addr.hpp @@ -33,6 +33,17 @@ union Addr { Addr(ip6::Addr addr) noexcept : ip6_{std::move(addr)} {} + // IP4 variant + Addr(uint8_t a, uint8_t b, uint8_t c, uint8_t d) + : ip4_{0, ip4_sign_be, {a, b, c, d}} + {} + + // IP6 variant + Addr(uint16_t a, uint16_t b, uint16_t c, uint16_t d, + uint16_t e, uint16_t f, uint16_t g, uint16_t h) + : ip6_{a, b, c, d, e, f, g, h} + {} + bool is_v4() const noexcept { return ip4_.big == 0 and ip4_.sign == ip4_sign_be; } diff --git a/api/net/inet.hpp b/api/net/inet.hpp index 856efc983f..159a4f202b 100644 --- a/api/net/inet.hpp +++ b/api/net/inet.hpp @@ -421,7 +421,7 @@ namespace net { { if (dest.is_loopback()) - return {0,0,0,1}; + return ip6::Addr{0,0,0,1}; if (is_loopback(dest)) return dest; diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index a7e2ee231b..b5d336170c 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -52,7 +52,7 @@ struct Addr { i16[6] = htons(d1); i16[7] = htons(d2); } - Addr(uint32_t a, uint32_t b, uint32_t c, uint32_t d) + explicit Addr(uint32_t a, uint32_t b, uint32_t c, uint32_t d) { i32[0] = htonl(a); i32[1] = htonl(b); i32[2] = htonl(c); i32[3] = htonl(d); diff --git a/test/net/unit/ip6_packet_test.cpp b/test/net/unit/ip6_packet_test.cpp index 68441eaa3f..786b735594 100644 --- a/test/net/unit/ip6_packet_test.cpp +++ b/test/net/unit/ip6_packet_test.cpp @@ -44,7 +44,7 @@ CASE("IP6 Packet HOPLIMIT - multiple packets") std::vector addrs{ {0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd}, {0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x83e7}, - {0,0,0,1}, + ip6::Addr{0,0,0,1}, {0xfe80, 0, 0, 0, 0x0202, 0xb3ff, 0xff1e, 0x8329}, }; diff --git a/test/net/unit/socket.cpp b/test/net/unit/socket.cpp index 69d7d72e33..12b4c5745b 100644 --- a/test/net/unit/socket.cpp +++ b/test/net/unit/socket.cpp @@ -36,7 +36,7 @@ CASE("Creating a Socket without arguments is empty") CASE("Creating a Socket with arguments is not empty") { - const Socket::Address addr { ip4::Addr{10,0,0,42} }; + const Socket::Address addr {10,0,0,42}; const Socket::port_t port = 80; Socket socket { addr, 80 }; @@ -51,15 +51,15 @@ CASE("Creating a Socket with arguments is not empty") CASE("Sockets can be compared to each other") { - Socket sock1 { ip4::Addr{ 10,0,0,42 }, 80 }; - Socket sock2 { ip4::Addr{ 10,0,0,42 }, 8080 }; - Socket sock3 { ip4::Addr{ 192,168,0,1 }, 80 }; + Socket sock1 { { 10,0,0,42 }, 80 }; + Socket sock2 { { 10,0,0,42 }, 8080 }; + Socket sock3 { { 192,168,0,1 }, 80 }; EXPECT_NOT( sock1 == sock2 ); EXPECT_NOT( sock1 == sock3 ); EXPECT( sock2 != sock3 ); - const Socket temp { ip4::Addr{ 10,0,0,42 }, 80 }; + const Socket temp { { 10,0,0,42 }, 80 }; EXPECT( sock1 == temp ); const Socket empty; From 53aa22e7a70c74cb303261537f209c458f97891e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 12 Jun 2018 16:02:45 +0200 Subject: [PATCH 416/723] tcp: Made Packet_view template to support non owning ptr storage --- api/net/tcp/common.hpp | 3 -- api/net/tcp/packet4_view.hpp | 28 +++++++++------- api/net/tcp/packet6_view.hpp | 29 ++++++++++------- api/net/tcp/packet_view.hpp | 62 ++++++++++++++++++++++-------------- 4 files changed, 72 insertions(+), 50 deletions(-) diff --git a/api/net/tcp/common.hpp b/api/net/tcp/common.hpp index 220b538567..d2b3b9837b 100644 --- a/api/net/tcp/common.hpp +++ b/api/net/tcp/common.hpp @@ -65,9 +65,6 @@ namespace net { return std::make_shared> (std::forward (args)...); } - class Packet_view; - using Packet_view_ptr = std::unique_ptr; - class Connection; using Connection_ptr = std::shared_ptr; diff --git a/api/net/tcp/packet4_view.hpp b/api/net/tcp/packet4_view.hpp index db23431c98..e07fe528e9 100644 --- a/api/net/tcp/packet4_view.hpp +++ b/api/net/tcp/packet4_view.hpp @@ -6,13 +6,18 @@ namespace net::tcp { -class Packet4_view : public Packet_view { +template class Packet4_v; +using Packet4_view = Packet4_v; +using Packet4_view_raw = Packet4_v; + +template +class Packet4_v : public Packet_v { public: - Packet4_view(std::unique_ptr ptr) - : Packet_view(std::move(ptr)) + Packet4_v(Ptr_type ptr) + : Packet_v(std::move(ptr)) { Expects(packet().is_ipv4()); - set_header(packet().ip_data().data()); + this->set_header(packet().ip_data().data()); } inline void init(); @@ -31,10 +36,10 @@ class Packet4_view : public Packet_view { private: PacketIP4& packet() noexcept - { return static_cast(*pkt); } + { return static_cast(*this->pkt); } const PacketIP4& packet() const noexcept - { return static_cast(*pkt); } + { return static_cast(*this->pkt); } void set_ip_src(const net::Addr& addr) noexcept override { packet().set_ip_src(addr.v4()); } @@ -56,15 +61,16 @@ class Packet4_view : public Packet_view { }; -inline void Packet4_view::init() +template +inline void Packet4_v::init() { // clear TCP header - memset(header, 0, sizeof(tcp::Header)); + memset(this->header, 0, sizeof(tcp::Header)); - set_win(tcp::default_window_size); - header->offset_flags.offset_reserved = (5 << 4); + this->set_win(tcp::default_window_size); + this->header->offset_flags.offset_reserved = (5 << 4); - set_length(); + this->set_length(); } } diff --git a/api/net/tcp/packet6_view.hpp b/api/net/tcp/packet6_view.hpp index cff5288fa4..f109e3f10b 100644 --- a/api/net/tcp/packet6_view.hpp +++ b/api/net/tcp/packet6_view.hpp @@ -6,13 +6,18 @@ namespace net::tcp { -class Packet6_view : public Packet_view { +template class Packet6_v; +using Packet6_view = Packet6_v; +using Packet6_view_raw = Packet6_v; + +template +class Packet6_v : public Packet_v { public: - Packet6_view(std::unique_ptr ptr) - : Packet_view(std::move(ptr)) + Packet6_v(std::unique_ptr ptr) + : Packet_v(std::move(ptr)) { Expects(packet().is_ipv6()); - set_header(packet().ip_data().data()); + this->set_header(packet().ip_data().data()); } inline void init(); @@ -31,10 +36,10 @@ class Packet6_view : public Packet_view { private: PacketIP6& packet() noexcept - { return static_cast(*pkt); } + { return static_cast(*this->pkt); } const PacketIP6& packet() const noexcept - { return static_cast(*pkt); } + { return static_cast(*this->pkt); } void set_ip_src(const net::Addr& addr) noexcept override { packet().set_ip_src(addr.v6()); } @@ -55,15 +60,15 @@ class Packet6_view : public Packet_view { { return packet().ip_header_len(); } }; - -inline void Packet6_view::init() +template +inline void Packet6_v::init() { // clear TCP header - memset(header, 0, sizeof(tcp::Header)); + memset(this->header, 0, sizeof(tcp::Header)); - set_win(tcp::default_window_size); - header->offset_flags.offset_reserved = (5 << 4); - set_length(); + this->set_win(tcp::default_window_size); + this->header->offset_flags.offset_reserved = (5 << 4); + this->set_length(); } } diff --git a/api/net/tcp/packet_view.hpp b/api/net/tcp/packet_view.hpp index 6efbf9fff6..42810452ba 100644 --- a/api/net/tcp/packet_view.hpp +++ b/api/net/tcp/packet_view.hpp @@ -11,7 +11,13 @@ namespace net::tcp { -class Packet_view { +template class Packet_v; +using Packet_view = Packet_v; +using Packet_view_raw = Packet_v; +using Packet_view_ptr = std::unique_ptr; + +template +class Packet_v { public: const Header& tcp_header() const noexcept @@ -31,20 +37,20 @@ class Packet_view { Socket destination() const noexcept { return Socket{ip_dst(), dst_port()}; } - Packet_view& set_src_port(port_t p) noexcept + Packet_v& set_src_port(port_t p) noexcept { tcp_header().source_port = htons(p); return *this; } - Packet_view& set_dst_port(port_t p) noexcept + Packet_v& set_dst_port(port_t p) noexcept { tcp_header().destination_port = htons(p); return *this; } - Packet_view& set_source(const Socket& src) + Packet_v& set_source(const Socket& src) { set_ip_src(src.address()); set_src_port(src.port()); return *this; } - Packet_view& set_destination(const Socket& dest) + Packet_v& set_destination(const Socket& dest) { set_ip_dst(dest.address()); set_dst_port(dest.port()); @@ -70,25 +76,25 @@ class Packet_view { { return tcp_header().offset_flags.offset_reserved >> 4; } - Packet_view& set_seq(seq_t n) noexcept + Packet_v& set_seq(seq_t n) noexcept { tcp_header().seq_nr = htonl(n); return *this; } - Packet_view& set_ack(seq_t n) noexcept + Packet_v& set_ack(seq_t n) noexcept { tcp_header().ack_nr = htonl(n); return *this; } - Packet_view& set_win(uint16_t size) noexcept + Packet_v& set_win(uint16_t size) noexcept { tcp_header().window_size = htons(size); return *this; } - Packet_view& set_flag(Flag f) noexcept + Packet_v& set_flag(Flag f) noexcept { tcp_header().offset_flags.whole |= htons(f); return *this; } - Packet_view& set_flags(uint16_t f) noexcept + Packet_v& set_flags(uint16_t f) noexcept { tcp_header().offset_flags.whole |= htons(f); return *this; } - Packet_view& clear_flag(Flag f) noexcept + Packet_v& clear_flag(Flag f) noexcept { tcp_header().offset_flags.whole &= ~ htons(f); return *this; } - Packet_view& clear_flags() + Packet_v& clear_flags() { tcp_header().offset_flags.whole &= 0x00ff; return *this; } // Set raw TCP offset in quadruples @@ -103,9 +109,12 @@ class Packet_view { uint16_t tcp_length() const { return tcp_header_length() + tcp_data_length(); } + uint16_t tcp_checksum() const noexcept + { return tcp_header().checksum; } + virtual uint16_t compute_tcp_checksum() const noexcept = 0; - Packet_view& set_tcp_checksum(uint16_t checksum) noexcept + Packet_v& set_tcp_checksum(uint16_t checksum) noexcept { tcp_header().checksum = checksum; return *this; } void set_tcp_checksum() noexcept @@ -188,26 +197,26 @@ class Packet_view { // Packet_view specific operations // - net::Packet_ptr release() + Ptr_type release() { Expects(pkt != nullptr && "Packet ptr is already null"); return std::move(pkt); } - const net::Packet_ptr& packet_ptr() const noexcept + const Ptr_type& packet_ptr() const noexcept { return pkt; } // hmm virtual Protocol ipv() const noexcept = 0; - virtual ~Packet_view() = default; + virtual ~Packet_v() = default; protected: - net::Packet_ptr pkt; - Header* header = nullptr; + Ptr_type pkt; + Header* header = nullptr; - Packet_view(net::Packet_ptr ptr) + Packet_v(Ptr_type ptr) : pkt{std::move(ptr)} { Expects(pkt != nullptr); @@ -245,8 +254,9 @@ inline unsigned round_up(unsigned n, unsigned div) { return (n + div - 1) / div; } +template template -inline void Packet_view::add_tcp_option(Args&&... args) { +inline void Packet_v::add_tcp_option(Args&&... args) { // to avoid headache, options need to be added BEFORE any data. assert(!has_tcp_data()); struct NOP { @@ -277,8 +287,9 @@ inline void Packet_view::add_tcp_option(Args&&... args) { set_length(); // update } +template template -inline void Packet_view::add_tcp_option_aligned(Args&&... args) { +inline void Packet_v::add_tcp_option_aligned(Args&&... args) { // to avoid headache, options need to be added BEFORE any data. Expects(!has_tcp_data()); @@ -298,7 +309,8 @@ inline void Packet_view::add_tcp_option_aligned(Args&&... args) { } // assumes the packet contains no other options. -inline const Option::opt_ts* Packet_view::parse_ts_option() noexcept +template +inline const Option::opt_ts* Packet_v::parse_ts_option() noexcept { auto* opt = this->tcp_options(); // TODO: improve by iterate option instead of byte (see Connection::parse_options) @@ -311,7 +323,8 @@ inline const Option::opt_ts* Packet_view::parse_ts_option() noexcept return this->ts_opt; } -inline size_t Packet_view::fill(const uint8_t* buffer, size_t length) +template +inline size_t Packet_v::fill(const uint8_t* buffer, size_t length) { size_t rem = ip_capacity() - tcp_length(); if(rem == 0) return 0; @@ -323,7 +336,8 @@ inline size_t Packet_view::fill(const uint8_t* buffer, size_t length) return total; } -inline std::string Packet_view::to_string() const +template +inline std::string Packet_v::to_string() const { char buffer[256]; int len = snprintf(buffer, sizeof(buffer), From 9eca389a47d051cd316ac98506da2f2082118051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 12 Jun 2018 16:04:34 +0200 Subject: [PATCH 417/723] net: Refactored NAT to make use of tcp Packet_view --- api/net/nat/nat.hpp | 12 ++-- src/net/nat/nat.cpp | 144 ++++++++++++++++++++++++++------------------ 2 files changed, 91 insertions(+), 65 deletions(-) diff --git a/api/net/nat/nat.hpp b/api/net/nat/nat.hpp index 0d08b28f44..c58a329fd1 100644 --- a/api/net/nat/nat.hpp +++ b/api/net/nat/nat.hpp @@ -26,32 +26,32 @@ namespace net { namespace nat { /* TCP Source NAT */ -void tcp_snat(PacketIP4& pkt, Socket new_sock); +void tcp_snat(PacketIP4& pkt, const Socket& new_sock); void tcp_snat(PacketIP4& pkt, const ip4::Addr new_addr); void tcp_snat(PacketIP4& pkt, const uint16_t new_port); /* UDP Source NAT */ -void udp_snat(PacketIP4& pkt, Socket new_sock); +void udp_snat(PacketIP4& pkt, const Socket& new_sock); void udp_snat(PacketIP4& pkt, const ip4::Addr new_addr); void udp_snat(PacketIP4& pkt, const uint16_t new_port); /* ICMP Source NAT */ void icmp_snat(PacketIP4& pkt, const ip4::Addr new_addr); /* IP4 Source NAT (depending on specified protocol) */ -void snat(PacketIP4& pkt, Socket src_socket); +void snat(PacketIP4& pkt, const Socket& src_socket); void snat(PacketIP4& pkt, const ip4::Addr new_addr); void snat(PacketIP4& pkt, const uint16_t new_port); /* TCP Destination NAT */ -void tcp_dnat(PacketIP4& pkt, Socket new_sock); +void tcp_dnat(PacketIP4& pkt, const Socket& new_sock); void tcp_dnat(PacketIP4& pkt, const ip4::Addr new_addr); void tcp_dnat(PacketIP4& pkt, const uint16_t new_port); /* UDP Destination NAT */ -void udp_dnat(PacketIP4& pkt, Socket new_sock); +void udp_dnat(PacketIP4& pkt, const Socket& new_sock); void udp_dnat(PacketIP4& pkt, const ip4::Addr new_addr); void udp_dnat(PacketIP4& pkt, const uint16_t new_port); /* ICMP Destination NAT */ void icmp_dnat(PacketIP4& pkt, const ip4::Addr new_addr); /* IP4 Destination NAT (depending on specified protocol) */ -void dnat(PacketIP4& pkt, Socket dst_socket); +void dnat(PacketIP4& pkt, const Socket& dst_socket); void dnat(PacketIP4& pkt, const ip4::Addr new_addr); void dnat(PacketIP4& pkt, const uint16_t new_port); diff --git a/src/net/nat/nat.cpp b/src/net/nat/nat.cpp index 24b5fc72fd..4baf74456b 100644 --- a/src/net/nat/nat.cpp +++ b/src/net/nat/nat.cpp @@ -17,7 +17,7 @@ #include #include // checksum_adjust -#include +#include #include namespace net { @@ -25,14 +25,13 @@ namespace nat { // Helper functions inline void recalc_ip_checksum(PacketIP4& pkt, ip4::Addr old_addr, ip4::Addr new_addr); -inline void recalc_tcp_sock(tcp::Packet& pkt, Socket osock, Socket nsock); -inline void recalc_tcp_addr(tcp::Packet& pkt, ip4::Addr old_addr, ip4::Addr new_addr); -inline void recalc_tcp_port(tcp::Packet& pkt, uint16_t old_port, uint16_t new_port); -inline void recalc_udp_sock(PacketUDP& pkt, Socket osock, Socket nsock); +inline void recalc_tcp_addr(tcp::Packet4_view_raw& pkt, ip4::Addr old_addr, ip4::Addr new_addr); +inline void recalc_tcp_port(tcp::Packet4_view_raw& pkt, uint16_t old_port, uint16_t new_port); +inline void recalc_udp_sock(PacketUDP& pkt, const Socket& osock, const Socket& nsock); inline void recalc_udp_addr(PacketUDP& pkt, ip4::Addr old_addr, ip4::Addr new_addr); inline void recalc_udp_port(PacketUDP& pkt, uint16_t old_port, uint16_t new_port); -void snat(PacketIP4& pkt, Socket src_socket) +void snat(PacketIP4& pkt, const Socket& src_socket) { switch(pkt.ip_protocol()) { @@ -87,7 +86,7 @@ void snat(PacketIP4& pkt, const uint16_t new_port) } } -void dnat(PacketIP4& pkt, Socket dst_socket) +void dnat(PacketIP4& pkt, const Socket& dst_socket) { switch(pkt.ip_protocol()) { @@ -143,30 +142,54 @@ void dnat(PacketIP4& pkt, const uint16_t new_port) } // TCP SNAT // -void tcp_snat(PacketIP4& p, Socket new_sock) +void tcp_snat(PacketIP4& ip4, const Socket& new_sock) { - Expects(p.ip_protocol() == Protocol::TCP); - auto& pkt = static_cast(p); - // Recalc checksum - recalc_tcp_sock(pkt, pkt.source(), new_sock); - // change source socket - pkt.set_source(new_sock); + Expects(ip4.ip_protocol() == Protocol::TCP); + + // IP4 source + auto old_addr = ip4.ip_src(); + auto new_addr = new_sock.address().v4(); + // recalc IP checksum + recalc_ip_checksum(ip4, old_addr, new_addr); + + // TCP + tcp::Packet4_view_raw pkt{&ip4}; + auto tcp_sum = pkt.tcp_checksum(); + // recalc tcp address part + checksum_adjust(&tcp_sum, &old_addr, &new_addr); + // recalc tcp port part + auto old_port = htons(pkt.src_port()); + auto new_port = htons(new_sock.port()); + checksum_adjust(&tcp_sum, &old_port, &new_port); + + // set the new sum + pkt.set_tcp_checksum(tcp_sum); + + // change source address and port + ip4.set_ip_src(new_addr); + pkt.set_src_port(new_sock.port()); } -void tcp_snat(PacketIP4& p, const ip4::Addr new_addr) +void tcp_snat(PacketIP4& ip4, const ip4::Addr new_addr) { - Expects(p.ip_protocol() == Protocol::TCP); - auto& pkt = static_cast(p); + Expects(ip4.ip_protocol() == Protocol::TCP); + + // recalc IP checksum + auto old_addr = ip4.ip_src(); + recalc_ip_checksum(ip4, old_addr, new_addr); + // recalc tcp address csum - recalc_tcp_addr(pkt, pkt.ip_src(), new_addr); + tcp::Packet4_view_raw pkt{&ip4}; + recalc_tcp_addr(pkt, old_addr, new_addr); + // change source address - pkt.set_ip_src(new_addr); + ip4.set_ip_src(new_addr); } void tcp_snat(PacketIP4& p, const uint16_t new_port) { Expects(p.ip_protocol() == Protocol::TCP); - auto& pkt = static_cast(p); + tcp::Packet4_view_raw pkt{&p}; // recalc tcp port recalc_tcp_port(pkt, pkt.src_port(), new_port); // change source port @@ -174,30 +197,54 @@ void tcp_snat(PacketIP4& p, const uint16_t new_port) } // TCP DNAT // -void tcp_dnat(PacketIP4& p, Socket new_sock) +void tcp_dnat(PacketIP4& ip4, const Socket& new_sock) { - Expects(p.ip_protocol() == Protocol::TCP); - auto& pkt = static_cast(p); - // Recalc checksum - recalc_tcp_sock(pkt, pkt.destination(), new_sock); - // change destination socket - pkt.set_destination(new_sock); + Expects(ip4.ip_protocol() == Protocol::TCP); + + // IP4 dest + auto old_addr = ip4.ip_dst(); + auto new_addr = new_sock.address().v4(); + // recalc IP checksum + recalc_ip_checksum(ip4, old_addr, new_addr); + + // TCP + tcp::Packet4_view_raw pkt{&ip4}; + auto tcp_sum = pkt.tcp_checksum(); + // recalc tcp address part + checksum_adjust(&tcp_sum, &old_addr, &new_addr); + // recalc tcp port part + auto old_port = htons(pkt.dst_port()); + auto new_port = htons(new_sock.port()); + checksum_adjust(&tcp_sum, &old_port, &new_port); + + // set the new sum + pkt.set_tcp_checksum(tcp_sum); + + // change source address and port + ip4.set_ip_dst(new_addr); + pkt.set_dst_port(new_sock.port()); } -void tcp_dnat(PacketIP4& p, const ip4::Addr new_addr) +void tcp_dnat(PacketIP4& ip4, const ip4::Addr new_addr) { - Expects(p.ip_protocol() == Protocol::TCP); - auto& pkt = static_cast(p); + Expects(ip4.ip_protocol() == Protocol::TCP); + + // recalc IP checksum + auto old_addr = ip4.ip_dst(); + recalc_ip_checksum(ip4, old_addr, new_addr); + // recalc tcp address csum - recalc_tcp_addr(pkt, pkt.ip_dst(), new_addr); + tcp::Packet4_view_raw pkt{&ip4}; + recalc_tcp_addr(pkt, old_addr, new_addr); + // change destination address - pkt.set_ip_dst(new_addr); + ip4.set_ip_dst(new_addr); } void tcp_dnat(PacketIP4& p, const uint16_t new_port) { Expects(p.ip_protocol() == Protocol::TCP); - auto& pkt = static_cast(p); + tcp::Packet4_view_raw pkt{&p}; // recalc tcp port csum recalc_tcp_port(pkt, pkt.dst_port(), new_port); // change destination port @@ -205,7 +252,7 @@ void tcp_dnat(PacketIP4& p, const uint16_t new_port) } // UDP SNAT // -void udp_snat(PacketIP4& p, Socket new_sock) +void udp_snat(PacketIP4& p, const Socket& new_sock) { Expects(p.ip_protocol() == Protocol::UDP); auto& pkt = static_cast(p); @@ -238,7 +285,7 @@ void udp_snat(PacketIP4& p, const uint16_t new_port) } // UDP DNAT // -void udp_dnat(PacketIP4& p, Socket new_sock) +void udp_dnat(PacketIP4& p, const Socket& new_sock) { Expects(p.ip_protocol() == Protocol::UDP); auto& pkt = static_cast(p); @@ -294,36 +341,15 @@ inline void recalc_ip_checksum(PacketIP4& pkt, ip4::Addr old_addr, ip4::Addr new pkt.set_ip_checksum(ip_sum); } -inline void recalc_tcp_sock(tcp::Packet& pkt, Socket osock, Socket nsock) -{ - auto old_addr = osock.address().v4(); - auto new_addr = nsock.address().v4(); - - // recalc IP checksum - recalc_ip_checksum(pkt, old_addr, new_addr); - - auto tcp_sum = pkt.tcp_checksum(); - // recalc tcp address part - checksum_adjust(&tcp_sum, &old_addr, &new_addr); - // recalc tcp port part - auto old_port = htons(osock.port()); - auto new_port = htons(nsock.port()); - checksum_adjust(&tcp_sum, &old_port, &new_port); - // set the new sum - pkt.set_tcp_checksum(tcp_sum); -} - -inline void recalc_tcp_addr(tcp::Packet& pkt, ip4::Addr old_addr, ip4::Addr new_addr) +inline void recalc_tcp_addr(tcp::Packet4_view_raw& pkt, ip4::Addr old_addr, ip4::Addr new_addr) { - // recalc IP checksum - recalc_ip_checksum(pkt, old_addr, new_addr); // recalc tcp address part auto tcp_sum = pkt.tcp_checksum(); checksum_adjust(&tcp_sum, &old_addr, &new_addr); pkt.set_tcp_checksum(tcp_sum); } -inline void recalc_tcp_port(tcp::Packet& pkt, uint16_t old_port, uint16_t new_port) +inline void recalc_tcp_port(tcp::Packet4_view_raw& pkt, uint16_t old_port, uint16_t new_port) { // swap ports to network order old_port = htons(old_port); @@ -334,7 +360,7 @@ inline void recalc_tcp_port(tcp::Packet& pkt, uint16_t old_port, uint16_t new_po pkt.set_tcp_checksum(tcp_sum); } -inline void recalc_udp_sock(PacketUDP& pkt, Socket osock, Socket nsock) +inline void recalc_udp_sock(PacketUDP& pkt, const Socket& osock, const Socket& nsock) { auto old_addr = osock.address().v4(); auto new_addr = nsock.address().v4(); From 1bba83ed1145ce9f62ccff26b7da170509a2317c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 13 Jun 2018 10:18:02 +0200 Subject: [PATCH 418/723] test: Updated some test to reflect new Addr design --- test/net/integration/nat/service.cpp | 3 ++- test/posix/integration/syslog_plugin/service.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/net/integration/nat/service.cpp b/test/net/integration/nat/service.cpp index a1aaf23997..3c1467b484 100644 --- a/test/net/integration/nat/service.cpp +++ b/test/net/integration/nat/service.cpp @@ -19,6 +19,7 @@ #include #include #include +#include using namespace net; @@ -162,7 +163,7 @@ void Service::start() server.tcp().listen(DNAT_PORT, [] (auto conn) { INFO("TCP DNAT", "Server received connection - %s", conn->to_string().c_str()); - CHECKSERT(conn->remote().address() != eth1.ip_addr(), + CHECKSERT(conn->remote().address().v4() != eth1.ip_addr(), "Received non SNAT connection - %s", conn->remote().to_string().c_str()); conn->on_read(1024, [conn](auto buf) diff --git a/test/posix/integration/syslog_plugin/service.cpp b/test/posix/integration/syslog_plugin/service.cpp index 988fdfae37..e7cc612683 100644 --- a/test/posix/integration/syslog_plugin/service.cpp +++ b/test/posix/integration/syslog_plugin/service.cpp @@ -50,7 +50,7 @@ int main() /* ------------------------- Testing IncludeOS syslog ------------------------- */ // Setting IP and port for the syslog messages Syslog::settings( {10, 0, 0, 2}, 6514 ); - printf("Syslog messages are sent to IP %s and port %d\n", Syslog::ip().str().c_str(), Syslog::port()); + printf("Syslog messages are sent to IP %s and port %d\n", Syslog::ip().to_string().c_str(), Syslog::port()); invalid_priority = -1; Syslog::syslog(invalid_priority, "Invalid %d", invalid_priority); From 89da7a909475ada6a549b63fa8f302227633d924 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 14 Jun 2018 11:08:11 +0200 Subject: [PATCH 419/723] Add manual_build.sh, a fast 64-bit build --- manual_build.sh | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100755 manual_build.sh diff --git a/manual_build.sh b/manual_build.sh new file mode 100755 index 0000000000..d532e74755 --- /dev/null +++ b/manual_build.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e +#export INCLUDEOS_PREFIX=$HOME/includeos + +# set default compiler if not set +CC=${CC:-clang-5.0} +CXX=${CXX:-clang++-5.0} + +read -rsp $'Press enter to continue...\n' + +# install chainloader +mkdir -p $INCLUDEOS_PREFIX/includeos +curl -L -o $INCLUDEOS_PREFIX/includeos/chainloader https://github.com/fwsGonzo/barebones/releases/download/v0.9-cl/chainloader + +# cleanup old build +rm -rf build_x86_64 + +mkdir -p build_x86_64 +pushd build_x86_64 +cmake .. -DCMAKE_INSTALL_PREFIX=$INCLUDEOS_PREFIX +make -j32 +make install +popd From 255cdfb6a9ac21ed1a47c6f53626e465603c48ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 14 Jun 2018 12:49:22 +0200 Subject: [PATCH 420/723] net: Temp packet include until NaCl is fixed --- api/net/tcp/tcp.hpp | 1 + test/net/integration/nat/service.cpp | 4 ++-- test/net/integration/udp/service.cpp | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index e7695fe0d9..358eaafd26 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -24,6 +24,7 @@ #include "headers.hpp" #include "listener.hpp" #include "packet_view.hpp" +#include "packet.hpp" // remove me, temp for NaCl #include // connections, listeners #include // writeq diff --git a/test/net/integration/nat/service.cpp b/test/net/integration/nat/service.cpp index 3c1467b484..31a364a59e 100644 --- a/test/net/integration/nat/service.cpp +++ b/test/net/integration/nat/service.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include using namespace net; @@ -141,7 +141,7 @@ void Service::start() if(pkt->ip_protocol() == Protocol::TCP) { - auto& tcp = static_cast(*pkt); + auto tcp = tcp::Packet4_view_raw(pkt.get()); if(tcp.dst_port() == DNAT_PORT) natty->dnat(*pkt, entry, SERVER); diff --git a/test/net/integration/udp/service.cpp b/test/net/integration/udp/service.cpp index 7f5624f1ce..4ee1ce913c 100644 --- a/test/net/integration/udp/service.cpp +++ b/test/net/integration/udp/service.cpp @@ -27,7 +27,7 @@ void Service::start() inet.network_config({ 10, 0, 0, 55 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 1 } ); // Gateway - printf("Service IP address is %s\n", inet.ip_addr().str().c_str()); + printf("Service IP address is %s\n", inet.ip_addr().to_string().c_str()); // UDP const UDP::port_t port = 4242; @@ -40,7 +40,7 @@ void Service::start() { std::string strdata(data, len); CHECK(1, "Getting UDP data from %s: %d -> %s", - addr.str().c_str(), port, strdata.c_str()); + addr.to_string().c_str(), port, strdata.c_str()); // send the same thing right back! using namespace std::chrono; Timers::oneshot(100ms, From dc5a31c0165a255b901879a7ebcbbde50f8839eb Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 14 Jun 2018 15:17:05 +0200 Subject: [PATCH 421/723] net: Use real link layer offset in Packet size determination --- api/hw/nic.hpp | 5 +---- api/net/link_layer.hpp | 6 +----- src/drivers/e1000.cpp | 4 ++-- src/drivers/e1000.hpp | 5 +---- src/drivers/virtionet.cpp | 6 +++--- src/drivers/virtionet.hpp | 5 +---- src/drivers/vmxnet3.cpp | 8 ++++---- src/drivers/vmxnet3.hpp | 5 +---- test/lest_util/nic_mock.hpp | 11 +++-------- 9 files changed, 17 insertions(+), 38 deletions(-) diff --git a/api/hw/nic.hpp b/api/hw/nic.hpp index 0d975babec..a9b4d72b80 100644 --- a/api/hw/nic.hpp +++ b/api/hw/nic.hpp @@ -77,11 +77,8 @@ namespace hw { size_t buffers_total() { return bufstore_.total_buffers(); } - /** Number of bytes in a frame needed by the device itself **/ - virtual size_t frame_offset_device() = 0; - /** Number of bytes in a frame needed by the link layer **/ - virtual size_t frame_offset_link() = 0; + virtual size_t frame_offset_link() const noexcept = 0; /** * Create a packet with appropriate size for the underlying link diff --git a/api/net/link_layer.hpp b/api/net/link_layer.hpp index a74c6dd49f..bfc8bbc644 100644 --- a/api/net/link_layer.hpp +++ b/api/net/link_layer.hpp @@ -60,12 +60,8 @@ class Link_layer : public hw::Nic { void set_vlan_upstream(upstream handler) override { link_.set_vlan_upstream(handler); } - /** Number of bytes in a frame needed by the device itself **/ - virtual size_t frame_offset_device() override - { return 0; } - /** Number of bytes in a frame needed by the linklayer **/ - virtual size_t frame_offset_link() override + size_t frame_offset_link() const noexcept override { return Protocol::header_size(); } hw::Nic::Proto proto() const override diff --git a/src/drivers/e1000.cpp b/src/drivers/e1000.cpp index a5372cabdb..c91308cc95 100644 --- a/src/drivers/e1000.cpp +++ b/src/drivers/e1000.cpp @@ -455,7 +455,7 @@ e1000::recv_packet(uint8_t* data, uint16_t size) new (ptr) net::Packet( DRIVER_OFFSET, size, - DRIVER_OFFSET + packet_len(), + DRIVER_OFFSET + size, &bufstore()); return net::Packet_ptr(ptr); } @@ -467,7 +467,7 @@ e1000::create_packet(int link_offset) new (ptr) net::Packet( DRIVER_OFFSET + link_offset, 0, - DRIVER_OFFSET + packet_len(), + DRIVER_OFFSET + frame_offset_link() + MTU(), buffer.bufstore); return net::Packet_ptr(ptr); } diff --git a/src/drivers/e1000.hpp b/src/drivers/e1000.hpp index 660598c171..beebdc909d 100644 --- a/src/drivers/e1000.hpp +++ b/src/drivers/e1000.hpp @@ -47,7 +47,7 @@ class e1000 : public net::Link_layer } uint16_t packet_len() const noexcept { - return sizeof(net::ethernet::VLAN_header) + MTU(); + return frame_offset_link() + MTU(); } net::downstream create_physical_downstream() override @@ -55,9 +55,6 @@ class e1000 : public net::Link_layer net::Packet_ptr create_packet(int) override; - size_t frame_offset_device() override - { return DRIVER_OFFSET; }; - /** Linklayer input. Hooks into IP-stack bottom, w.DOWNSTREAM data.*/ void transmit(net::Packet_ptr pckt); diff --git a/src/drivers/virtionet.cpp b/src/drivers/virtionet.cpp index bd566cd2f7..8afa9fb768 100644 --- a/src/drivers/virtionet.cpp +++ b/src/drivers/virtionet.cpp @@ -285,7 +285,7 @@ void VirtioNet::add_receive_buffer(uint8_t* pkt) auto* vnet = pkt + sizeof(Packet); Token token1 {{vnet, sizeof(virtio_net_hdr)}, Token::IN }; - Token token2 {{vnet + sizeof(virtio_net_hdr), packet_len()}, Token::IN }; + Token token2 {{vnet + sizeof(virtio_net_hdr), max_packet_len()}, Token::IN }; std::array tokens {{ token1, token2 }}; rx_q.enqueue(tokens); @@ -299,7 +299,7 @@ VirtioNet::recv_packet(uint8_t* data, uint16_t size) new (ptr) net::Packet( sizeof(virtio_net_hdr), size - sizeof(virtio_net_hdr), - sizeof(virtio_net_hdr) + packet_len(), + size, &bufstore()); return net::Packet_ptr(ptr); @@ -314,7 +314,7 @@ VirtioNet::create_packet(int link_offset) new (ptr) net::Packet( sizeof(virtio_net_hdr) + link_offset, 0, - sizeof(virtio_net_hdr) + packet_len(), + sizeof(virtio_net_hdr) + frame_offset_link() + MTU(), buffer.bufstore); return net::Packet_ptr(ptr); diff --git a/src/drivers/virtionet.hpp b/src/drivers/virtionet.hpp index ae7612e16c..ad7db44d05 100644 --- a/src/drivers/virtionet.hpp +++ b/src/drivers/virtionet.hpp @@ -133,15 +133,12 @@ class VirtioNet : Virtio, public net::Link_layer { uint16_t MTU() const noexcept override { return 1500; } - uint16_t packet_len() const noexcept { + uint16_t max_packet_len() const noexcept { return sizeof(net::ethernet::VLAN_header) + MTU(); } net::Packet_ptr create_packet(int) override; - size_t frame_offset_device() override - { return sizeof(virtio_net_hdr); }; - net::downstream create_physical_downstream() override { return {this, &VirtioNet::transmit}; } diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index 2323b38a79..bcb9f30cc4 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -231,7 +231,7 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : shared.misc.queue_desc_address = (uintptr_t) &dma->queues; shared.misc.driver_data_len = sizeof(vmxnet3_dma); shared.misc.queue_desc_len = sizeof(vmxnet3_queues); - shared.misc.mtu = packet_len(); // 60-9000 + shared.misc.mtu = max_packet_len(); // 60-9000 shared.misc.num_tx_queues = 1; shared.misc.num_rx_queues = NUM_RX_QUEUES; shared.interrupt.mask_mode = VMXNET3_IT_AUTO | (VMXNET3_IMM_AUTO << 2); @@ -359,7 +359,7 @@ void vmxnet3::refill(rxring_state& rxq) // assign rx descriptor auto& desc = rxq.desc0[i]; desc.address = (uintptr_t) rxq.buffers[i]; - desc.flags = packet_len() | generation; + desc.flags = max_packet_len() | generation; rxq.prod_count++; rxq.producers++; } @@ -377,7 +377,7 @@ vmxnet3::recv_packet(uint8_t* data, uint16_t size) new (ptr) net::Packet( DRIVER_OFFSET, size, - DRIVER_OFFSET + packet_len(), + DRIVER_OFFSET + size, &bufstore()); return net::Packet_ptr(ptr); } @@ -389,7 +389,7 @@ vmxnet3::create_packet(int link_offset) new (ptr) net::Packet( DRIVER_OFFSET + link_offset, 0, - DRIVER_OFFSET + packet_len(), + DRIVER_OFFSET + frame_offset_link() + MTU(), buffer.bufstore); return net::Packet_ptr(ptr); } diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index 59fdad8dbe..c81ad0b942 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -50,7 +50,7 @@ class vmxnet3 : public net::Link_layer return m_mtu; } - uint16_t packet_len() const noexcept { + uint16_t max_packet_len() const noexcept { return sizeof(net::ethernet::VLAN_header) + MTU(); } @@ -59,9 +59,6 @@ class vmxnet3 : public net::Link_layer net::Packet_ptr create_packet(int) override; - size_t frame_offset_device() override - { return DRIVER_OFFSET; }; - /** Linklayer input. Hooks into IP-stack bottom, w.DOWNSTREAM data.*/ void transmit(net::Packet_ptr pckt); diff --git a/test/lest_util/nic_mock.hpp b/test/lest_util/nic_mock.hpp index 1c8cf099b0..84dd1fa530 100644 --- a/test/lest_util/nic_mock.hpp +++ b/test/lest_util/nic_mock.hpp @@ -71,12 +71,8 @@ class Nic_mock : public hw::Nic { void set_vlan_upstream(upstream handler) override { vlan_handler_ = handler; } - /** Number of bytes in a frame needed by the device itself **/ - size_t frame_offset_device() override - { return frame_offs_dev_; } - /** Number of bytes in a frame needed by the link layer **/ - size_t frame_offset_link() override + size_t frame_offset_link() const noexcept override { return frame_offs_link_; } static constexpr uint16_t packet_len() @@ -87,9 +83,9 @@ class Nic_mock : public hw::Nic { auto buffer = bufstore().get_buffer(); auto* ptr = (net::Packet*) buffer.addr; - new (ptr) net::Packet(frame_offs_dev_ + frame_offs_link_, + new (ptr) net::Packet(frame_offs_link_, 0, - frame_offs_dev_ + packet_len(), + packet_len(), buffer.bufstore); return net::Packet_ptr(ptr); @@ -123,7 +119,6 @@ class Nic_mock : public hw::Nic { // Public data members (ahem) MAC::Addr mac_ = {0xc0,0x00,0x01,0x70,0x00,0x01}; - static constexpr size_t frame_offs_dev_ = 0; static constexpr size_t frame_offs_link_ = 14; std::vector tx_queue_; From ae111b8e8ecab5052590b29e3f2a88052f056034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 15 Jun 2018 10:45:54 +0200 Subject: [PATCH 422/723] tcp: Enable packet6 view for raw ptrs as well --- api/net/tcp/packet6_view.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/net/tcp/packet6_view.hpp b/api/net/tcp/packet6_view.hpp index f109e3f10b..c7108a8c5b 100644 --- a/api/net/tcp/packet6_view.hpp +++ b/api/net/tcp/packet6_view.hpp @@ -13,7 +13,7 @@ using Packet6_view_raw = Packet6_v; template class Packet6_v : public Packet_v { public: - Packet6_v(std::unique_ptr ptr) + Packet6_v(Ptr_type ptr) : Packet_v(std::move(ptr)) { Expects(packet().is_ipv6()); From b5c37102f774b53be4cf5e9770c66eb74b257de8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 15 Jun 2018 10:46:38 +0200 Subject: [PATCH 423/723] ip4: Silence unused var in debug --- src/net/ip4/ip4.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/net/ip4/ip4.cpp b/src/net/ip4/ip4.cpp index ae0ed5d8ef..404c5cafc5 100644 --- a/src/net/ip4/ip4.cpp +++ b/src/net/ip4/ip4.cpp @@ -106,7 +106,7 @@ namespace net { or local_ip() == ADDR_ANY; } - void IP4::receive(Packet_ptr pckt, const bool /*link_bcast*/) + void IP4::receive(Packet_ptr pckt, [[maybe_unused]]const bool link_bcast) { // Cast to IP4 Packet auto packet = static_unique_ptr_cast(std::move(pckt)); @@ -305,7 +305,8 @@ namespace net { // Stat increment packets transmitted packets_tx_++; - PRINT(" Transmitting packet, layer begin: buf + %li\n", packet->layer_begin() - packet->buf()); + PRINT(" Transmitting packet, layer begin: buf + %li ip.len=%u pkt.size=%zu\n", + packet->layer_begin() - packet->buf(), packet->ip_total_length(), packet->size()); linklayer_out_(std::move(packet), next_hop); } From 31d2685b572d3e63ae7dac1edbc98f36d2d7830d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 15 Jun 2018 10:47:29 +0200 Subject: [PATCH 424/723] test: TCP test is now IPv6 --- test/net/integration/tcp/CMakeLists.txt | 2 ++ test/net/integration/tcp/service.cpp | 35 +++++++++++++++---------- test/net/integration/tcp/test.py | 25 ++++++++++-------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/test/net/integration/tcp/CMakeLists.txt b/test/net/integration/tcp/CMakeLists.txt index 07d1769957..747ea73374 100644 --- a/test/net/integration/tcp/CMakeLists.txt +++ b/test/net/integration/tcp/CMakeLists.txt @@ -28,6 +28,8 @@ set(SOURCES set(DRIVERS virtionet # Virtio networking + vmxnet3 + e1000 # virtioblock # Virtio block device # ... Others from IncludeOS/src/drivers ) diff --git a/test/net/integration/tcp/service.cpp b/test/net/integration/tcp/service.cpp index b8bed4aa36..fd0a4c2548 100644 --- a/test/net/integration/tcp/service.cpp +++ b/test/net/integration/tcp/service.cpp @@ -16,9 +16,8 @@ // limitations under the License. #include -#include -#include -#include +#include +#include #include #include #include @@ -27,6 +26,9 @@ using namespace net; using namespace std::chrono; // For timers and MSL tcp::Connection_ptr client; +static Inet& stack() +{ return Super_stack::get(0); } + /* TEST VARIABLES */ @@ -58,16 +60,16 @@ void FINISH_TEST() { Timers::oneshot(2 * MSL_TEST + 100ms, [] (Timers::id_t) { INFO("TEST", "Verify release of resources"); - CHECKSERT(Inet::stack<0>().tcp().active_connections() == 0, + CHECKSERT(stack().tcp().active_connections() == 0, "No (0) active connections"); INFO("Buffers", "%u avail / %u total", - Inet::stack<0>().buffers_available(), - Inet::stack<0>().buffers_total()); + stack().buffers_available(), + stack().buffers_total()); // unfortunately we can't know just how many buffers SHOULD be // available, because drivers take some buffers, but there should // be at least half the buffers left - auto total = Inet::stack<0>().buffers_total(); - CHECKSERT(Inet::stack<0>().buffers_available() >= total / 2, + auto total = stack().buffers_total(); + CHECKSERT(stack().buffers_available() >= total / 2, "Free buffers (%u available)", total); printf("# TEST SUCCESS #\n"); }); @@ -80,13 +82,13 @@ void OUTGOING_TEST_INTERNET(const HostAddress& address) { auto port = address.second; // This needs correct setup to work INFO("TEST", "Outgoing Internet Connection (%s:%u)", address.first.c_str(), address.second); - Inet::stack<0>().resolve(address.first, + stack().resolve(address.first, [port](auto ip_address, const Error&) { CHECK(ip_address != 0, "Resolved host"); if(ip_address != 0) { - Inet::stack<0>().tcp().connect({ip_address, port}) + stack().tcp().connect({ip_address, port}) ->on_connect([](tcp::Connection_ptr conn) { CHECKSERT(conn != nullptr, "Connected"); @@ -104,7 +106,7 @@ void OUTGOING_TEST_INTERNET(const HostAddress& address) { */ void OUTGOING_TEST(Socket outgoing) { INFO("TEST", "Outgoing Connection (%s)", outgoing.to_string().c_str()); - Inet::stack<0>().tcp().connect(outgoing, [](tcp::Connection_ptr conn) + stack().tcp().connect(outgoing, [](tcp::Connection_ptr conn) { CHECKSERT(conn != nullptr, "Connection successfully established."); conn->on_read(small.size(), @@ -151,13 +153,18 @@ void Service::start() for(int i = 0; i < H; i++) huge += TEST_STR; huge += "-end"; - auto& inet = Inet::stack<0>(); // Inet::stack<0>(); + auto& inet = stack(); inet.network_config( { 10, 0, 0, 44 }, // IP { 255,255,255, 0 }, // Netmask { 10, 0, 0, 1 }, // Gateway { 8, 8, 8, 8 } // DNS ); + inet.network_config6( + { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd }, // IP6 + 64, // Prefix6 + { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x83e7 } // Gateway6 + ); INFO("Buffers available", "%u", inet.buffers_available()); auto& tcp = inet.tcp(); @@ -246,7 +253,7 @@ void Service::start() tcp.listen(TEST4).on_connect([](tcp::Connection_ptr conn) { INFO("Test 4","Connection/TCP state"); // There should be at least one connection. - CHECKSERT(Inet::stack<0>().tcp().active_connections() > 0, "There is (>0) open connection(s)"); + CHECKSERT(stack().tcp().active_connections() > 0, "There is (>0) open connection(s)"); // Test if connected. CHECKSERT(conn->is_connected(), "Is connected"); // Test if writable. @@ -264,7 +271,7 @@ void Service::start() [conn] (auto) { CHECKSERT(conn->is_state({"TIME-WAIT"}), "State: TIME-WAIT"); INFO("Test 4", "Succeeded. Trigger TEST5"); - OUTGOING_TEST({Inet::stack().gateway(), TEST5}); + OUTGOING_TEST({Inet::stack().gateway6(), TEST5}); }); Timers::oneshot(5s, [] (Timers::id_t) { FINISH_TEST(); }); diff --git a/test/net/integration/tcp/test.py b/test/net/integration/tcp/test.py index cfe68fb6be..95982c503c 100755 --- a/test/net/integration/tcp/test.py +++ b/test/net/integration/tcp/test.py @@ -11,9 +11,8 @@ from vmrunner import vmrunner from vmrunner.prettify import color -# Usage: python test.py $GUEST_IP $HOST_IP -GUEST = '10.0.0.44' if (len(sys.argv) < 2) else sys.argv[1] -HOST = '10.0.0.1' if (len(sys.argv) < 3) else sys.argv[2] +GUEST = 'fe80:0:0:0:e823:fcff:fef4:85bd%bridge43' +HOST = 'fe80:0:0:0:e823:fcff:fef4:83e7%bridge43' TEST1 = 8081 @@ -25,10 +24,12 @@ INFO = color.INFO("") def connect(port): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_address = (GUEST, port) - print INFO, 'connecting to %s port %s' % server_address - sock.connect(server_address) + addr = (GUEST, port) + res = socket.getaddrinfo(addr[0], addr[1], socket.AF_INET6, socket.SOCK_STREAM, socket.SOL_TCP) + af, socktype, proto, canonname, sa = res[0] + sock = socket.socket(af, socktype, proto) + print INFO, 'connecting to %s' % res + sock.connect(sa) bytes_received = 0 try: while True: @@ -46,11 +47,13 @@ def connect(port): return True def listen(port): - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + addr = (HOST, port) + res = socket.getaddrinfo(addr[0], addr[1], socket.AF_INET6, socket.SOCK_STREAM, socket.SOL_TCP) + print INFO, 'starting up on %s' % res + af, socktype, proto, canonname, sa = res[0] + sock = socket.socket(af, socktype, proto) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - server_address = (HOST, port) - print INFO, 'starting up on %s port %s' % server_address - sock.bind(server_address) + sock.bind(sa) sock.listen(1) while True: From bc611444ea992840f55f8459bf441b8b569d4fb1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 15 Jun 2018 11:25:00 +0200 Subject: [PATCH 425/723] linux: Add IP6 NDP module --- linux/userspace/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linux/userspace/CMakeLists.txt b/linux/userspace/CMakeLists.txt index f78d4665fc..3f52c7a710 100644 --- a/linux/userspace/CMakeLists.txt +++ b/linux/userspace/CMakeLists.txt @@ -12,7 +12,7 @@ set(NET_SOURCES ${IOS}/src/net/ip4/reassembly.cpp ${IOS}/src/net/ip6/ip6.cpp ${IOS}/src/net/ip6/icmp6.cpp - #${IOS}/src/net/ip6/ndp.cpp + ${IOS}/src/net/ip6/ndp.cpp ${IOS}/src/net/tcp/tcp.cpp ${IOS}/src/net/tcp/connection.cpp From dfca7b1b94c6c28705aad17b4a03e46ded5f28a6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 15 Jun 2018 11:25:25 +0200 Subject: [PATCH 426/723] cmake: Update Botan to latest master --- cmake/botan.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/botan.cmake b/cmake/botan.cmake index b5a3581e69..e5dbaebe2f 100644 --- a/cmake/botan.cmake +++ b/cmake/botan.cmake @@ -4,14 +4,14 @@ include(ExternalProject) if(${ARCH} STREQUAL "x86_64") - set(BOTAN_HASH b1dfea0d906783e00dc650832d47ffd4) + set(BOTAN_HASH 6f0a3e4aaf6723d83fd025176249372b) elseif(${ARCH} STREQUAL "i686") - set(BOTAN_HASH e49e7177fe015de294acb0c8dc13a099) + set(BOTAN_HASH 0175670a61b45a04f832ee9e5965f1f3) endif() ExternalProject_Add(botan PREFIX botan - URL https://github.com/includeos/botan/releases/download/musl-1.0/botan-includeos-${ARCH}.tar.gz + URL https://github.com/includeos/botan/releases/download/musl-1.1/botan-includeos-${ARCH}.tar.gz URL_HASH MD5=${BOTAN_HASH} CONFIGURE_COMMAND "" BUILD_COMMAND "" From 8b73faed8721696da0b9772b14729c909cc3c8b6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 15 Jun 2018 11:26:24 +0200 Subject: [PATCH 427/723] test: Add Linux US variant for icmp6 integration test --- cmake/linux.service.cmake | 3 ++ .../integration/icmp6/linux/CMakeLists.txt | 21 ++++++++++ test/net/integration/icmp6/linux/run.sh | 39 +++++++++++++++++++ test/net/integration/icmp6/service.cpp | 7 +++- 4 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 test/net/integration/icmp6/linux/CMakeLists.txt create mode 100755 test/net/integration/icmp6/linux/run.sh diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index bdd80587ff..c223053c8d 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -99,3 +99,6 @@ if (STRIP_BINARY) DEPENDS service ) endif() + +# write binary name to file +file(WRITE ${CMAKE_BINARY_DIR}/binary.txt ${BINARY}) diff --git a/test/net/integration/icmp6/linux/CMakeLists.txt b/test/net/integration/icmp6/linux/CMakeLists.txt new file mode 100644 index 0000000000..04820eb746 --- /dev/null +++ b/test/net/integration/icmp6/linux/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 2.8.9) +if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) + set(ENV{INCLUDEOS_PREFIX} /usr/local) +endif() +project (service C CXX) + +# Human-readable name of your service +set(SERVICE_NAME "Linux userspace ICMP6 test") + +# Name of your service binary +set(BINARY "icmp6") + +# Source files to be linked with OS library parts to form bootable image +set(SOURCES + ../service.cpp + ) + +set(EXTRA_LIBS + ) + +include($ENV{INCLUDEOS_PREFIX}/includeos/linux.service.cmake) diff --git a/test/net/integration/icmp6/linux/run.sh b/test/net/integration/icmp6/linux/run.sh new file mode 100755 index 0000000000..5fe9056068 --- /dev/null +++ b/test/net/integration/icmp6/linux/run.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +export GPROF="OFF" +export DBG="OFF" + +export num_jobs=${num_jobs:--j32} + +function make_linux(){ + pushd $INCLUDEOS_SRC/linux/build + cmake -DGPROF=$GPROF -DDEBUGGING=$DBG .. + make $num_jobs install + popd +} + +function make_service(){ + mkdir -p build + pushd build + cmake -DGPROF=$GPROF -DDEBUGGING=$DBG .. + make $num_jobs + popd +} + +if [ -z "$1" ] +then + GPROF="OFF" +elif [ "$1" == "gprof" ] +then + GPROF="ON" +fi + + +make_linux +make_service + +#sudo mknod /dev/net/tap c 10 200 +BINARY=build/"`cat build/binary.txt`" +sudo $BINARY diff --git a/test/net/integration/icmp6/service.cpp b/test/net/integration/icmp6/service.cpp index 9884ebe948..f8dba3039e 100644 --- a/test/net/integration/icmp6/service.cpp +++ b/test/net/integration/icmp6/service.cpp @@ -20,8 +20,13 @@ using namespace net; -void Service::start(const std::string&) +void Service::start() { +#ifdef USERSPACE_LINUX + extern void create_network_device(int N, const char* route, const char* ip); + create_network_device(0, "10.0.0.0/24", "10.0.0.1"); +#endif + auto& inet = Inet::stack<0>(); inet.network_config({ 10, 0, 0, 52 }, // IP { 255, 255, 0, 0 }, // Netmask From 3dd5c67f97ee0d3bf073d06d68411b633753d712 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 15 Jun 2018 12:02:10 +0200 Subject: [PATCH 428/723] mana: Fix TCP component not building --- .../include/mana/components/dashboard/components/tcp.hpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/mana/include/mana/components/dashboard/components/tcp.hpp b/lib/mana/include/mana/components/dashboard/components/tcp.hpp index 1f6764c29e..2ae4e739f5 100644 --- a/lib/mana/include/mana/components/dashboard/components/tcp.hpp +++ b/lib/mana/include/mana/components/dashboard/components/tcp.hpp @@ -42,7 +42,7 @@ class TCP : public Component { writer.StartObject(); writer.Key("address"); - writer.String(tcp_.address().str()); + writer.String(tcp_.address().to_string()); writer.Key("ifname"); writer.String(tcp_.stack().ifname()); @@ -120,7 +120,3 @@ class TCP : public Component { } // < namespace mana #endif - - - - From 9b7adbb149b8db09f44c16599d774a7a5239a236 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 18 Jun 2018 16:48:09 +0200 Subject: [PATCH 429/723] linux: Update to latest, better strip --- cmake/linux.service.cmake | 6 +----- linux/src/drivers/usernet.hpp | 3 --- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index c223053c8d..7ca3c7d09e 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -93,11 +93,7 @@ if (ENABLE_LTO OR USE_LLD) endif() if (STRIP_BINARY) - add_custom_target( - strip_binary ALL - COMMAND strip --strip-all ${BINARY} - DEPENDS service - ) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -s") endif() # write binary name to file diff --git a/linux/src/drivers/usernet.hpp b/linux/src/drivers/usernet.hpp index 7003b91c97..49c94902c1 100644 --- a/linux/src/drivers/usernet.hpp +++ b/linux/src/drivers/usernet.hpp @@ -47,9 +47,6 @@ class UserNet : public net::Link_layer { net::Packet_ptr create_packet(int) override; - size_t frame_offset_device() override - { return sizeof(driver_hdr); }; - net::downstream create_physical_downstream() override { return {this, &UserNet::transmit}; } From 8762b91f4e54aea7b4656a8be6fb74089a21e68e Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 18 Jun 2018 16:50:15 +0200 Subject: [PATCH 430/723] cmake: Dont strip binary by default --- cmake/post.service.cmake | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index b909f70d88..ce866b2816 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -242,12 +242,9 @@ set(CMAKE_CXX_LINK_EXECUTABLE " -o set(BUILD_SHARED_LIBRARIES OFF) set(CMAKE_EXE_LINKER_FLAGS "-static") -set(STRIP_LV) +set(LD_STRIP) if (NOT debug) - set(STRIP_LV "--strip-debug") -endif() -if (stripped) - set(STRIP_LV "--strip-all") + set(LD_STRIP "--strip-debug") endif() set(ELF ${ARCH}) @@ -261,7 +258,7 @@ if ("${PLATFORM}" STREQUAL "x86_solo5") set(PRE_BSS_SIZE "--defsym PRE_BSS_AREA=0x200000") endif() -set(LDFLAGS "-nostdlib -melf_${ELF} --eh-frame-hdr ${STRIP_LV} --script=${INSTALL_LOC}/${ARCH}/linker.ld ${PRE_BSS_SIZE}") +set(LDFLAGS "-nostdlib -melf_${ELF} --eh-frame-hdr ${LD_STRIP} --script=${INSTALL_LOC}/${ARCH}/linker.ld ${PRE_BSS_SIZE}") set_target_properties(service PROPERTIES LINK_FLAGS "${LDFLAGS}") @@ -479,21 +476,19 @@ target_link_libraries(service # write binary location to known file file(WRITE ${CMAKE_BINARY_DIR}/binary.txt ${BINARY}) -set(STRIP_LV ${CMAKE_STRIP} --strip-all ${BINARY}) -if (debug) - unset(STRIP_LV) -endif() - +# old behavior: remote all symbols after elfsym if (NOT debug) - add_custom_target( - pruned_elf_symbols ALL - COMMAND ${INSTALL_LOC}/bin/elf_syms ${BINARY} - COMMAND ${CMAKE_OBJCOPY} --update-section .elf_symbols=_elf_symbols.bin ${BINARY} ${BINARY} - COMMAND ${STRIP_LV} - DEPENDS service - ) + set(STRIP_LV ${CMAKE_STRIP} --strip-all ${BINARY}) endif() +add_custom_target( + pruned_elf_symbols ALL + COMMAND ${INSTALL_LOC}/bin/elf_syms ${BINARY} + COMMAND ${CMAKE_OBJCOPY} --update-section .elf_symbols=_elf_symbols.bin ${BINARY} ${BINARY} + #COMMAND ${STRIP_LV} + DEPENDS service + ) + # create bare metal .img: make legacy_bootloader add_custom_target( legacy_bootloader From 8ab9557e439cd8d1e9ccf2c08cfffc9bd86d1466 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 18 Jun 2018 17:49:02 +0200 Subject: [PATCH 431/723] net: Decouple bufferstore from public APIs --- api/hw/nic.hpp | 18 ++---------------- api/net/inet.hpp | 7 ------- api/net/link_layer.hpp | 6 +++--- api/net/vif.hpp | 2 +- linux/src/drivers/usernet.cpp | 3 +-- src/drivers/e1000.cpp | 2 +- src/drivers/e1000.hpp | 2 ++ src/drivers/solo5net.cpp | 6 +----- src/drivers/solo5net.hpp | 7 ++----- src/drivers/virtionet.cpp | 2 +- src/drivers/virtionet.hpp | 2 ++ src/drivers/vmxnet3.cpp | 2 +- src/drivers/vmxnet3.hpp | 2 ++ test/lest_util/nic_mock.hpp | 4 ++-- test/linux/websockets/run.sh | 3 ++- test/net/integration/tcp/service.cpp | 10 ---------- 16 files changed, 23 insertions(+), 55 deletions(-) diff --git a/api/hw/nic.hpp b/api/hw/nic.hpp index a9b4d72b80..6d34cb603d 100644 --- a/api/hw/nic.hpp +++ b/api/hw/nic.hpp @@ -18,7 +18,6 @@ #ifndef HW_NIC_HPP #define HW_NIC_HPP -#include "../net/buffer_store.hpp" #include "mac_addr.hpp" #include @@ -66,17 +65,6 @@ namespace hw { virtual void set_arp_upstream(upstream handler) = 0; virtual void set_vlan_upstream(upstream handler) = 0; - net::BufferStore& bufstore() noexcept - { return bufstore_; } - - /** Number of free buffers in the BufferStore **/ - size_t buffers_available() - { return bufstore_.available(); } - - /** Number of total buffers in the BufferStore **/ - size_t buffers_total() - { return bufstore_.total_buffers(); } - /** Number of bytes in a frame needed by the link layer **/ virtual size_t frame_offset_link() const noexcept = 0; @@ -107,7 +95,7 @@ namespace hw { virtual ~Nic() {} - /** Trigger a read from buffers, pusing any packets up the stack */ + /** Check for completed rx and pass rx packets up the stack */ virtual void poll() = 0; /** Overridable MTU detection function per-network **/ @@ -119,8 +107,7 @@ namespace hw { * * Constructed by the actual Nic Driver */ - Nic(net::BufferStore& bufstore) - : bufstore_{bufstore} + Nic() { static int id_counter = 0; N = id_counter++; @@ -147,7 +134,6 @@ namespace hw { } private: - net::BufferStore& bufstore_; int N; friend class Devices; }; diff --git a/api/net/inet.hpp b/api/net/inet.hpp index 159a4f202b..2a436935e8 100644 --- a/api/net/inet.hpp +++ b/api/net/inet.hpp @@ -310,13 +310,6 @@ namespace net { return nic_.transmit_queue_available(); } - size_t buffers_available() { - return nic_.buffers_available(); - } - size_t buffers_total() { - return nic_.buffers_total(); - } - void force_start_send_queues(); void move_to_this_cpu(); diff --git a/api/net/link_layer.hpp b/api/net/link_layer.hpp index bfc8bbc644..dc1acc22f6 100644 --- a/api/net/link_layer.hpp +++ b/api/net/link_layer.hpp @@ -30,7 +30,7 @@ class Link_layer : public hw::Nic { using upstream = hw::Nic::upstream; using downstream_link = hw::Nic::downstream; public: - explicit Link_layer(Protocol&& protocol, BufferStore& bufstore); + explicit Link_layer(Protocol&& protocol); std::string device_name() const override { return link_.link_name(); @@ -88,8 +88,8 @@ class Link_layer : public hw::Nic { }; template -Link_layer::Link_layer(Protocol&& protocol, BufferStore& bufstore) - : hw::Nic(bufstore), +Link_layer::Link_layer(Protocol&& protocol) + : hw::Nic(), link_{std::forward(protocol)} { } diff --git a/api/net/vif.hpp b/api/net/vif.hpp index c618d06802..cb3ddada47 100644 --- a/api/net/vif.hpp +++ b/api/net/vif.hpp @@ -32,7 +32,7 @@ class Vif : public Link_layer { using Linklayer = Link_layer; Vif(hw::Nic& link, const int id) - : Linklayer(Protocol{{this, &Vif::transmit}, link.mac(), id}, link.bufstore()), + : Linklayer(Protocol{{this, &Vif::transmit}, link.mac(), id}), link_{link}, id_{id}, phys_down_{link_.create_physical_downstream()} {} diff --git a/linux/src/drivers/usernet.cpp b/linux/src/drivers/usernet.cpp index 50352914e7..83b3a642b6 100644 --- a/linux/src/drivers/usernet.cpp +++ b/linux/src/drivers/usernet.cpp @@ -4,7 +4,7 @@ constexpr MAC::Addr UserNet::MAC_ADDRESS; UserNet::UserNet(const uint16_t mtu) - : Link(Link_protocol{{this, &UserNet::transmit}, mac()}, buffer_store), + : Link(Link_protocol{{this, &UserNet::transmit}, mac()}), mtu_value(mtu), buffer_store(256u, 128 + mtu) {} UserNet& UserNet::create(const uint16_t mtu) @@ -78,4 +78,3 @@ net::Packet_ptr UserNet::create_packet(int link_offset) return net::Packet_ptr(ptr); } -// wrap buffer-length (that should belong to bufferstore) in packet wrapper diff --git a/src/drivers/e1000.cpp b/src/drivers/e1000.cpp index c91308cc95..81b56feb80 100644 --- a/src/drivers/e1000.cpp +++ b/src/drivers/e1000.cpp @@ -65,7 +65,7 @@ static inline uint16_t buffer_size_for_mtu(const uint16_t mtu) #define NUM_PACKET_BUFFERS (NUM_TX_DESC + NUM_RX_DESC + NUM_TX_QUEUE + 8) e1000::e1000(hw::PCI_Device& d, uint16_t mtu) : - Link(Link_protocol{{this, &e1000::transmit}, mac()}, bufstore_), + Link(Link_protocol{{this, &e1000::transmit}, mac()}), m_pcidev(d), m_mtu(mtu), bufstore_{NUM_PACKET_BUFFERS, buffer_size_for_mtu(mtu)} { static_assert((NUM_RX_DESC * sizeof(rx_desc)) % 128 == 0, "Ring length must be 128-byte aligned"); diff --git a/src/drivers/e1000.hpp b/src/drivers/e1000.hpp index beebdc909d..403d316501 100644 --- a/src/drivers/e1000.hpp +++ b/src/drivers/e1000.hpp @@ -67,6 +67,8 @@ class e1000 : public net::Link_layer return free_transmit_descr() + NUM_TX_QUEUE - sendq_size; } + auto& bufstore() noexcept { return bufstore_; } + void flush() override; void deactivate() override; diff --git a/src/drivers/solo5net.cpp b/src/drivers/solo5net.cpp index 8961729c00..3b564f2b73 100644 --- a/src/drivers/solo5net.cpp +++ b/src/drivers/solo5net.cpp @@ -15,10 +15,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#define PRINT_INFO -#define DEBUG // Allow debuging -#define DEBUG2 - #include "solo5net.hpp" #include #include @@ -34,7 +30,7 @@ using namespace net; const char* Solo5Net::driver_name() const { return "Solo5Net"; } Solo5Net::Solo5Net() - : Link(Link_protocol{{this, &Solo5Net::transmit}, mac()}, bufstore_), + : Link(Link_protocol{{this, &Solo5Net::transmit}, mac()}), packets_rx_{Statman::get().create(Stat::UINT64, device_name() + ".packets_rx").get_uint64()}, packets_tx_{Statman::get().create(Stat::UINT64, device_name() + ".packets_tx").get_uint64()}, bufstore_{NUM_BUFFERS, 2048u} // don't change this diff --git a/src/drivers/solo5net.hpp b/src/drivers/solo5net.hpp index 8ac632734b..bfa7ecd29f 100644 --- a/src/drivers/solo5net.hpp +++ b/src/drivers/solo5net.hpp @@ -70,13 +70,10 @@ class Solo5Net : public net::Link_layer { return 1000; // any big random number for now } - /** Number of incoming packets waiting in the RX-queue */ - size_t receive_queue_waiting() { - return 0; - } - net::Packet_ptr create_packet(int) override; + auto& bufstore() noexcept { return bufstore_; } + void move_to_this_cpu() override {}; void deactivate() override; diff --git a/src/drivers/virtionet.cpp b/src/drivers/virtionet.cpp index 8afa9fb768..99a1b54680 100644 --- a/src/drivers/virtionet.cpp +++ b/src/drivers/virtionet.cpp @@ -62,7 +62,7 @@ void VirtioNet::get_config() { VirtioNet::VirtioNet(hw::PCI_Device& d, const uint16_t /*mtu*/) : Virtio(d), - Link(Link_protocol{{this, &VirtioNet::transmit}, mac()}, bufstore_), + Link(Link_protocol{{this, &VirtioNet::transmit}, mac()}), m_pcidev(d), bufstore_{VNET_TOT_BUFFERS(), 2048 /* half-page buffers */}, packets_rx_{Statman::get().create(Stat::UINT64, diff --git a/src/drivers/virtionet.hpp b/src/drivers/virtionet.hpp index ad7db44d05..51721b6144 100644 --- a/src/drivers/virtionet.hpp +++ b/src/drivers/virtionet.hpp @@ -156,6 +156,8 @@ class VirtioNet : Virtio, public net::Link_layer { bool link_up() const noexcept; + auto& bufstore() noexcept { return bufstore_; } + void deactivate() override; void flush() override { diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index bcb9f30cc4..5a725a85df 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -117,7 +117,7 @@ static inline uint16_t buffer_size_for_mtu(const uint16_t mtu) } vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : - Link(Link_protocol{{this, &vmxnet3::transmit}, mac()}, bufstore_), + Link(Link_protocol{{this, &vmxnet3::transmit}, mac()}), m_pcidev(d), m_mtu(mtu), bufstore_{1024, buffer_size_for_mtu(mtu)} { INFO("vmxnet3", "Driver initializing (rev=%#x)", d.rev_id()); diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index c81ad0b942..a1ea2d552c 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -70,6 +70,8 @@ class vmxnet3 : public net::Link_layer return tx_tokens_free(); } + auto& bufstore() noexcept { return bufstore_; } + void flush() override; void deactivate() override; diff --git a/test/lest_util/nic_mock.hpp b/test/lest_util/nic_mock.hpp index 84dd1fa530..5f32f018d4 100644 --- a/test/lest_util/nic_mock.hpp +++ b/test/lest_util/nic_mock.hpp @@ -80,7 +80,7 @@ class Nic_mock : public hw::Nic { net::Packet_ptr create_packet(int) override { - auto buffer = bufstore().get_buffer(); + auto buffer = bufstore_.get_buffer(); auto* ptr = (net::Packet*) buffer.addr; new (ptr) net::Packet(frame_offs_link_, @@ -115,7 +115,7 @@ class Nic_mock : public hw::Nic { // ~Nic_mock() {} - Nic_mock() : Nic(bufstore_), bufstore_{256u, 2048} {} + Nic_mock() : Nic(), bufstore_{256u, 2048} {} // Public data members (ahem) MAC::Addr mac_ = {0xc0,0x00,0x01,0x70,0x00,0x01}; diff --git a/test/linux/websockets/run.sh b/test/linux/websockets/run.sh index e345227f87..5fe9056068 100755 --- a/test/linux/websockets/run.sh +++ b/test/linux/websockets/run.sh @@ -35,4 +35,5 @@ make_linux make_service #sudo mknod /dev/net/tap c 10 200 -sudo ./build/websockets +BINARY=build/"`cat build/binary.txt`" +sudo $BINARY diff --git a/test/net/integration/tcp/service.cpp b/test/net/integration/tcp/service.cpp index fd0a4c2548..19c1e7c213 100644 --- a/test/net/integration/tcp/service.cpp +++ b/test/net/integration/tcp/service.cpp @@ -62,15 +62,6 @@ void FINISH_TEST() { INFO("TEST", "Verify release of resources"); CHECKSERT(stack().tcp().active_connections() == 0, "No (0) active connections"); - INFO("Buffers", "%u avail / %u total", - stack().buffers_available(), - stack().buffers_total()); - // unfortunately we can't know just how many buffers SHOULD be - // available, because drivers take some buffers, but there should - // be at least half the buffers left - auto total = stack().buffers_total(); - CHECKSERT(stack().buffers_available() >= total / 2, - "Free buffers (%u available)", total); printf("# TEST SUCCESS #\n"); }); } @@ -165,7 +156,6 @@ void Service::start() 64, // Prefix6 { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x83e7 } // Gateway6 ); - INFO("Buffers available", "%u", inet.buffers_available()); auto& tcp = inet.tcp(); // reduce test duration From fb1a1cb73e02da0bb950e35b204f57bc3ba23e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Tue, 19 Jun 2018 16:35:53 +0200 Subject: [PATCH 432/723] common: Use panic instead of exit on Expects/Ensures --- api/common | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/common b/api/common index 25663a5792..2dc452b642 100644 --- a/api/common +++ b/api/common @@ -57,6 +57,7 @@ static_assert(sizeof(void*) == 8, "Pointer must match arch"); #include #endif +#include inline void __expect_fail(const char *expr, const char *file, int line, const char *func){ #ifndef INCLUDEOS_SINGLE_THREADED SMP::global_lock(); @@ -66,7 +67,7 @@ inline void __expect_fail(const char *expr, const char *file, int line, const ch #ifndef INCLUDEOS_SINGLE_THREADED SMP::global_unlock(); #endif - exit(67); + panic(expr); } #define Expects(x) ((void)((x) || (__expect_fail("Expects failed: "#x, __FILE__, __LINE__, __func__),0))) From 5cd780575ca466c1b75fb9c3fedd27015b7b04bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Tue, 19 Jun 2018 17:15:39 +0200 Subject: [PATCH 433/723] icmp6: set_payload -> add_payload, make payload static std::array --- api/net/ip6/icmp6.hpp | 7 ------- api/net/ip6/packet_icmp6.hpp | 10 +++++----- src/net/ip6/icmp6.cpp | 18 +++++++++++++----- src/net/ip6/ndp.cpp | 8 ++++---- 4 files changed, 22 insertions(+), 21 deletions(-) diff --git a/api/net/ip6/icmp6.hpp b/api/net/ip6/icmp6.hpp index d5e2117a3a..f82d878b2d 100644 --- a/api/net/ip6/icmp6.hpp +++ b/api/net/ip6/icmp6.hpp @@ -165,13 +165,6 @@ namespace net Stack& inet_; Ndp ndp_; downstream network_layer_out_ = nullptr; - uint8_t includeos_payload_[48] = {'I','N','C','L','U','D', - 'E','O','S','1','2','3','4','5', - 'A','B','C','D','E','F','G','H', - 'I','J','K','L','M','N','O','P', - 'Q','R','S','T','U','V','W','X', - 'Y','Z','1','2','3','4','5','6', - '7','8'}; inline bool is_full_header(size_t pckt_size) { return (pckt_size >= sizeof(IP6::header) + icmp6::Packet::header_size()); } diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index fc97a36e3e..9f3613d5f1 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -201,8 +201,8 @@ namespace icmp6 { header.type = type; header.len = len; - icmp6_.set_payload({reinterpret_cast(&header), - sizeof header}); + icmp6_.add_payload(reinterpret_cast(&header), + sizeof header); } uint8_t* get_option_data(int opt) @@ -372,10 +372,10 @@ namespace icmp6 { header().checksum = compute_checksum();; } - void set_payload(Span new_load) + void add_payload(const void* new_load, const size_t len) { - pckt_->set_data_end(pckt_->ip_header_len() + header_size() + payload_offset_ + new_load.size()); - memcpy(payload().data(), new_load.data(), payload().size()); + pckt_->set_data_end(pckt_->ip_header_len() + header_size() + payload_offset_ + len); + memcpy(payload().data(), new_load, payload().size()); payload_offset_ += payload().size(); } diff --git a/src/net/ip6/icmp6.cpp b/src/net/ip6/icmp6.cpp index e821879f3a..fc9099d49d 100644 --- a/src/net/ip6/icmp6.cpp +++ b/src/net/ip6/icmp6.cpp @@ -32,6 +32,13 @@ namespace net { + static constexpr std::array includeos_payload = + {'I','N','C','L','U','D','E','O', + 'S','1','2','3','4','5','A','B', + 'C','D','E','F','G','H','I','J', + 'K','L','M','N','O','P','Q','R', + 'S','T','U','V','W','X','Y','Z', + '1','2','3','4','5','6','7','8'}; // ---------------------------- ICMP_view ---------------------------- std::string ICMP6_view::to_string() const { @@ -225,9 +232,10 @@ namespace net PRINT(" Transmitting request to %s\n", dest_ip.to_string().c_str()); - // Payload - // Default: includeos_payload_ - req.set_payload(icmp6::Packet::Span(includeos_payload_, 68)); + //printf(" Request size: %i payload size: %i\n", + // req.ip().size(), req.payload().size()); + // Default payload + req.add_payload(includeos_payload.data(), includeos_payload.size()); // Add checksum req.set_checksum(); @@ -261,7 +269,7 @@ namespace net // Payload // Default: Header and 66 bits (8 bytes) of original payload - res.set_payload(req.header_and_data()); + res.add_payload(req.header_and_data().data(), req.header_and_data().size()); // Add checksum res.set_checksum(); @@ -302,7 +310,7 @@ namespace net res.ip().ip_dst().str().c_str()); // Payload - res.set_payload(req.payload()); + res.add_payload(req.payload().data(), req.payload().size()); // Add checksum res.set_checksum(); diff --git a/src/net/ip6/ndp.cpp b/src/net/ip6/ndp.cpp index cb62f8a989..a27d807bee 100644 --- a/src/net/ip6/ndp.cpp +++ b/src/net/ip6/ndp.cpp @@ -66,9 +66,9 @@ namespace net res.ndp().set_neighbour_adv_flag(NEIGH_ADV_SOL | NEIGH_ADV_OVERRIDE); // Insert target link address, ICMP6 option header and our mac address - res.set_payload({req.ndp().neighbour_sol().get_target().data(), 16 }); + res.add_payload(req.ndp().neighbour_sol().get_target().data(), 16); res.ndp().set_ndp_options_header(icmp6::ND_OPT_TARGET_LL_ADDR, 0x01); - res.set_payload({reinterpret_cast (&link_mac_addr()), 6}); + res.add_payload(reinterpret_cast (&link_mac_addr()), 6); // Add checksum res.set_checksum(); @@ -135,9 +135,9 @@ namespace net req.ip().set_ip_dst(dest_ip.solicit(target)); // Set target address - req.set_payload({target.data(), 16}); + req.add_payload(target.data(), 16); req.ndp().set_ndp_options_header(icmp6::ND_OPT_SOURCE_LL_ADDR, 0x01); - req.set_payload({reinterpret_cast (&link_mac_addr()), 6}); + req.add_payload(reinterpret_cast (&link_mac_addr()), 6); req.set_checksum(); From 4beeaf539702a9f60121b97a7d6ba914cb94e0c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Tue, 19 Jun 2018 17:16:10 +0200 Subject: [PATCH 434/723] icmp6: Add temp print statement showing payload size --- src/net/ip6/icmp6.cpp | 4 ++-- test/net/integration/tcp/service.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/net/ip6/icmp6.cpp b/src/net/ip6/icmp6.cpp index fc9099d49d..7a41fe20bd 100644 --- a/src/net/ip6/icmp6.cpp +++ b/src/net/ip6/icmp6.cpp @@ -232,8 +232,8 @@ namespace net PRINT(" Transmitting request to %s\n", dest_ip.to_string().c_str()); - //printf(" Request size: %i payload size: %i\n", - // req.ip().size(), req.payload().size()); + printf(" Request size: %i payload size: %i\n", + req.ip().size(), req.payload().size()); // Default payload req.add_payload(includeos_payload.data(), includeos_payload.size()); diff --git a/test/net/integration/tcp/service.cpp b/test/net/integration/tcp/service.cpp index fd0a4c2548..9846a81736 100644 --- a/test/net/integration/tcp/service.cpp +++ b/test/net/integration/tcp/service.cpp @@ -161,7 +161,7 @@ void Service::start() { 8, 8, 8, 8 } // DNS ); inet.network_config6( - { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd }, // IP6 + { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x93bd }, // IP6 64, // Prefix6 { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x83e7 } // Gateway6 ); From 80aaef11c5fea0b49a4afc441e44f228980c0519 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 20 Jun 2018 14:24:27 +0200 Subject: [PATCH 435/723] test: Add Linux US support to TCP test --- test/net/integration/tcp/linux/CMakeLists.txt | 21 ++++++++++ test/net/integration/tcp/linux/run.sh | 39 +++++++++++++++++++ test/net/integration/tcp/service.cpp | 5 +++ 3 files changed, 65 insertions(+) create mode 100644 test/net/integration/tcp/linux/CMakeLists.txt create mode 100755 test/net/integration/tcp/linux/run.sh diff --git a/test/net/integration/tcp/linux/CMakeLists.txt b/test/net/integration/tcp/linux/CMakeLists.txt new file mode 100644 index 0000000000..ec2ac6d29e --- /dev/null +++ b/test/net/integration/tcp/linux/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 2.8.9) +if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) + set(ENV{INCLUDEOS_PREFIX} /usr/local) +endif() +project (service C CXX) + +# Human-readable name of your service +set(SERVICE_NAME "Linux userspace TCP test") + +# Name of your service binary +set(BINARY "tcp_test") + +# Source files to be linked with OS library parts to form bootable image +set(SOURCES + ../service.cpp + ) + +set(EXTRA_LIBS + ) + +include($ENV{INCLUDEOS_PREFIX}/includeos/linux.service.cmake) diff --git a/test/net/integration/tcp/linux/run.sh b/test/net/integration/tcp/linux/run.sh new file mode 100755 index 0000000000..5fe9056068 --- /dev/null +++ b/test/net/integration/tcp/linux/run.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +export GPROF="OFF" +export DBG="OFF" + +export num_jobs=${num_jobs:--j32} + +function make_linux(){ + pushd $INCLUDEOS_SRC/linux/build + cmake -DGPROF=$GPROF -DDEBUGGING=$DBG .. + make $num_jobs install + popd +} + +function make_service(){ + mkdir -p build + pushd build + cmake -DGPROF=$GPROF -DDEBUGGING=$DBG .. + make $num_jobs + popd +} + +if [ -z "$1" ] +then + GPROF="OFF" +elif [ "$1" == "gprof" ] +then + GPROF="ON" +fi + + +make_linux +make_service + +#sudo mknod /dev/net/tap c 10 200 +BINARY=build/"`cat build/binary.txt`" +sudo $BINARY diff --git a/test/net/integration/tcp/service.cpp b/test/net/integration/tcp/service.cpp index 6c693df2e8..b377dbe391 100644 --- a/test/net/integration/tcp/service.cpp +++ b/test/net/integration/tcp/service.cpp @@ -134,6 +134,11 @@ struct Buffer { void Service::start() { +#ifdef USERSPACE_LINUX + extern void create_network_device(int N, const char* route, const char* ip); + create_network_device(0, "10.0.0.0/24", "10.0.0.1"); +#endif + for(int i = 0; i < S; i++) small += TEST_STR; big += "start-"; From 0cdd9cc71610ded00addea0e9cfb55e64268fbb1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 20 Jun 2018 14:28:20 +0200 Subject: [PATCH 436/723] net: Cleanup in packet.hpp --- api/net/packet.hpp | 56 ++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/api/net/packet.hpp b/api/net/packet.hpp index a4b3639177..6a11f184ec 100644 --- a/api/net/packet.hpp +++ b/api/net/packet.hpp @@ -114,28 +114,11 @@ namespace net data_end_ += i; } - /* Add a packet to this packet chain. */ - void chain(Packet_ptr p) noexcept { - if (!chain_) { - chain_ = std::move(p); - last_ = chain_.get(); - } else { - auto* ptr = p.get(); - last_->chain(std::move(p)); - last_ = ptr->last_in_chain() ? ptr->last_in_chain() : ptr; - assert(last_); - } - } + /* Add a packet to this packet chain */ + inline void chain(Packet_ptr p) noexcept; - int chain_length() const noexcept { - int count = 1; - auto* p = this; - while (p->chain_) { - p = p->chain_.get(); - count++; - } - return count; - } + /* Count packets in chain */ + inline int chain_length() const noexcept; /* Get the last packet in the chain */ Packet* last_in_chain() noexcept @@ -161,9 +144,6 @@ namespace net } private: - Packet_ptr chain_ = nullptr; - Packet* last_ = nullptr; - /** Set layer begin, e.g. view the packet from another layer */ void set_layer_begin(Byte_ptr loc) { @@ -184,10 +164,38 @@ namespace net Byte_ptr layer_begin_; Byte_ptr data_end_; const Byte* const buffer_end_; + + Packet_ptr chain_ = nullptr; + Packet* last_ = nullptr; + BufferStore* bufstore_; Byte buf_[0]; }; //< class Packet + void Packet::chain(Packet_ptr p) noexcept + { + if (!chain_) { + chain_ = std::move(p); + last_ = chain_.get(); + } else { + auto* ptr = p.get(); + last_->chain(std::move(p)); + last_ = ptr->last_in_chain() ? ptr->last_in_chain() : ptr; + assert(last_); + } + } + + int Packet::chain_length() const noexcept + { + int count = 1; + auto* p = this; + while (p->chain_) { + p = p->chain_.get(); + count++; + } + return count; + } + } //< namespace net #endif From d650e93456fc2f17038d2d33c8bbada61e6ef78b Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 20 Jun 2018 14:37:22 +0200 Subject: [PATCH 437/723] net: Add optional payload offset to Packet --- api/net/packet.hpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/api/net/packet.hpp b/api/net/packet.hpp index 6a11f184ec..0abc570051 100644 --- a/api/net/packet.hpp +++ b/api/net/packet.hpp @@ -101,6 +101,16 @@ namespace net set_layer_begin(layer_begin() + i); } + /** Modify/retrieve payload offset, used by higher levelprotocols */ + void set_payload_offset(int offset) noexcept + { + Expects(offset >= 0 and layer_begin() + offset < buffer_end_); + this->payload_off_ = layer_begin() + offset; + } + Byte_ptr payload() const noexcept { + return this->payload_off_; + } + /** Set data end / write-position relative to layer_begin */ void set_data_end(int offset) { @@ -163,6 +173,7 @@ namespace net Byte_ptr layer_begin_; Byte_ptr data_end_; + Byte_ptr payload_off_ = 0; const Byte* const buffer_end_; Packet_ptr chain_ = nullptr; From c26f4a3054e396be8a2bdc87ebefc947255b9aab Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 20 Jun 2018 17:06:16 +0200 Subject: [PATCH 438/723] ip6: Use payload in ipv6 packet, fixing #1814 --- api/net/ip6/packet_ip6.hpp | 19 ++++--------------- src/net/ip6/ip6.cpp | 5 ++++- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/api/net/ip6/packet_ip6.hpp b/api/net/ip6/packet_ip6.hpp index 04d3ee0319..ae6113a698 100644 --- a/api/net/ip6/packet_ip6.hpp +++ b/api/net/ip6/packet_ip6.hpp @@ -103,7 +103,7 @@ namespace net /* This returns the IPv6 header and extension header len. * Note: Extension header needs to be parsed to know this */ uint16_t ip_header_len() const noexcept - { return IP6_HEADER_LEN + get_extension_header_len(); } + { return payload() - layer_begin(); } // IPv6 setters // @@ -161,6 +161,7 @@ namespace net hdr.hop_limit = DEFAULT_HOP_LIMIT; hdr.next_header = static_cast(proto); increment_data_end(IP6_HEADER_LEN); + set_payload_offset(IP6_HEADER_LEN); } Span ip_data() { @@ -171,14 +172,6 @@ namespace net return {ip_data_ptr(), ip_data_length()}; } - void update_extension_header_len(uint8_t len) { - extension_header_len_ += len; - } - - uint16_t get_extension_header_len() const { - return extension_header_len_; - } - /** * Set IP6 payload length */ @@ -190,23 +183,19 @@ namespace net /** Get pointer to IP data */ Byte* ip_data_ptr() noexcept __attribute__((assume_aligned(4))) { - return layer_begin() + IP6_HEADER_LEN + get_extension_header_len(); + return payload(); } const Byte* ip_data_ptr() const noexcept __attribute__((assume_aligned(4))) { - return layer_begin() + IP6_HEADER_LEN + get_extension_header_len(); + return payload(); } - - private: const ip6::Header& ip6_header() const noexcept { return *reinterpret_cast(layer_begin()); } - uint16_t extension_header_len_; - }; //< class PacketIP6 } //< namespace net #endif diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index c37177c1c8..642937e1b0 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -96,6 +96,7 @@ namespace net auto next_proto = packet.next_protocol(); ip6::ExtensionHeader& ext = *(ip6::ExtensionHeader*)reader; uint8_t ext_len; + uint16_t pl_off = IP6_HEADER_LEN; while (next_proto != Protocol::IPv6_NONXT) { if (next_proto == Protocol::HOPOPT) { @@ -108,9 +109,11 @@ namespace net ext = *(ip6::ExtensionHeader*)reader; ext_len = ext.size(); reader += ext_len; - packet.update_extension_header_len(ext_len); + //packet.update_extension_header_len(ext_len); + pl_off += ext_len; next_proto = ext.next(); } + packet.set_payload_offset(pl_off); return next_proto; } From 97c7c72f8d54dd5e094952377b367de869afd6f4 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 20 Jun 2018 17:07:13 +0200 Subject: [PATCH 439/723] ip6: Avoid copy assignment of ext headers --- src/net/ip6/ip6.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index 642937e1b0..c261906327 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -94,7 +94,6 @@ namespace net { auto reader = packet.layer_begin() + IP6_HEADER_LEN; auto next_proto = packet.next_protocol(); - ip6::ExtensionHeader& ext = *(ip6::ExtensionHeader*)reader; uint8_t ext_len; uint16_t pl_off = IP6_HEADER_LEN; @@ -106,7 +105,7 @@ namespace net PRINT("Done parsing extension header, next proto: %d\n", next_proto); return next_proto; } - ext = *(ip6::ExtensionHeader*)reader; + auto& ext = *(ip6::ExtensionHeader*)reader; ext_len = ext.size(); reader += ext_len; //packet.update_extension_header_len(ext_len); From 005bc02648b23c256dc0ffd0bc3db1addfd098ad Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 21 Jun 2018 16:52:09 +0200 Subject: [PATCH 440/723] net: Add payload_length, increment_payload to Packet --- api/net/packet.hpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/api/net/packet.hpp b/api/net/packet.hpp index 0abc570051..f09865d507 100644 --- a/api/net/packet.hpp +++ b/api/net/packet.hpp @@ -104,12 +104,19 @@ namespace net /** Modify/retrieve payload offset, used by higher levelprotocols */ void set_payload_offset(int offset) noexcept { - Expects(offset >= 0 and layer_begin() + offset < buffer_end_); + Expects(offset >= 0 and layer_begin() + offset < buffer_end()); this->payload_off_ = layer_begin() + offset; } + void increment_payload_offset(int offset) noexcept { + this->payload_off_ += offset; + Expects(payload_off_ >= layer_begin() and payload_off_ < buffer_end()); + } Byte_ptr payload() const noexcept { return this->payload_off_; } + int payload_length() const noexcept { + return this->data_end() - payload(); + } /** Set data end / write-position relative to layer_begin */ void set_data_end(int offset) From dc5c27f1bb6176285552b3ddfb1092230cb6083f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 21 Jun 2018 16:53:02 +0200 Subject: [PATCH 441/723] ip6: Parse payload offset properly IP6/ICMP6 --- api/net/ip6/packet_icmp6.hpp | 31 +++++++++++++------------ api/net/ip6/packet_ip6.hpp | 18 ++++++++------- src/net/ip6/icmp6.cpp | 9 +++----- src/net/ip6/ip6.cpp | 34 +++++++++++++++++++++++----- test/net/integration/tcp/service.cpp | 2 +- 5 files changed, 59 insertions(+), 35 deletions(-) diff --git a/api/net/ip6/packet_icmp6.hpp b/api/net/ip6/packet_icmp6.hpp index 9f3613d5f1..93df51d90d 100644 --- a/api/net/ip6/packet_icmp6.hpp +++ b/api/net/ip6/packet_icmp6.hpp @@ -229,10 +229,10 @@ namespace icmp6 { }__attribute__((packed)); Header& header() - { return *reinterpret_cast(pckt_->layer_begin() + pckt_->ip_header_len()); } + { return *reinterpret_cast(pckt_->payload()); } const Header& header() const - { return *reinterpret_cast(pckt_->layer_begin() + pckt_->ip_header_len()); } + { return *reinterpret_cast(pckt_->payload()); } struct pseudo_header { @@ -266,18 +266,21 @@ namespace icmp6 { { return header().idse.sequence; } uint16_t payload_len() const noexcept - { return pckt_->data_end() - (pckt_->layer_begin() + pckt_->ip_header_len() + header_size()); } + { return pckt_->size() - (pckt_->ip_header_len() + header_size()); } /** * Where the payload of an ICMP packet starts, calculated from the start of the IP header * The payload of an ICMP error message packet contains the original packet sent that caused an * ICMP error to occur (the original IP header and 8 bytes of the original packet's data) */ - int payload_index() + int payload_index() const { return pckt_->ip_header_len() + header_size(); } Span payload() - { return {&(header().payload[0 + payload_offset_]), pckt_->data_end() - &(header().payload[0 + payload_offset_]) }; } + { + auto* icmp_pl = &header().payload[payload_offset_]; + return {icmp_pl, pckt_->data_end() - icmp_pl }; + } /** Several ICMP messages require the payload to be the header and 64 bits of the * data of the original datagram @@ -374,23 +377,23 @@ namespace icmp6 { void add_payload(const void* new_load, const size_t len) { - pckt_->set_data_end(pckt_->ip_header_len() + header_size() + payload_offset_ + len); - memcpy(payload().data(), new_load, payload().size()); - payload_offset_ += payload().size(); + pckt_->increment_data_end(len); + memcpy(payload().data(), new_load, len); + payload_offset_ += len; } /** Get the underlying IP packet */ - IP6::IP_packet& ip() const - { return *pckt_.get(); } + IP6::IP_packet& ip() { return *pckt_; } + const IP6::IP_packet& ip() const { return *pckt_; } /** Construct from existing packet **/ Packet(IP6::IP_packet_ptr pckt) - : pckt_{ std::move(pckt) }, ndp_(*this), payload_offset_{0} - { } + : pckt_{ std::move(pckt) }, ndp_(*this) + { } /** Provision fresh packet from factory **/ Packet(IP6::IP_packet_factory create) - : pckt_{create(Protocol::ICMPv6)}, ndp_(*this), payload_offset_{0} + : pckt_ { create(Protocol::ICMPv6) }, ndp_(*this) { pckt_->increment_data_end(sizeof(Header)); } @@ -405,7 +408,7 @@ namespace icmp6 { private: IP6::IP_packet_ptr pckt_; NdpPacket ndp_; - uint16_t payload_offset_; + uint16_t payload_offset_ = 0; }; } } diff --git a/api/net/ip6/packet_ip6.hpp b/api/net/ip6/packet_ip6.hpp index ae6113a698..d6602f0f78 100644 --- a/api/net/ip6/packet_ip6.hpp +++ b/api/net/ip6/packet_ip6.hpp @@ -35,10 +35,6 @@ namespace net // IPv6 header getters - /** Get IP protocol version field. Must be 6 */ - ip6::Header& ip6_header() noexcept - { return *reinterpret_cast(layer_begin()); } - uint32_t ver_tc_fl() const noexcept { return ip6_header().ver_tc_fl; } @@ -103,7 +99,10 @@ namespace net /* This returns the IPv6 header and extension header len. * Note: Extension header needs to be parsed to know this */ uint16_t ip_header_len() const noexcept - { return payload() - layer_begin(); } + { + assert(payload() != nullptr); + return payload() - layer_begin(); + } // IPv6 setters // @@ -155,15 +154,17 @@ namespace net } void init(Protocol proto = Protocol::HOPOPT) noexcept { - Expects(size() == 0); auto& hdr = ip6_header(); hdr = {}; hdr.hop_limit = DEFAULT_HOP_LIMIT; hdr.next_header = static_cast(proto); increment_data_end(IP6_HEADER_LEN); set_payload_offset(IP6_HEADER_LEN); + assert(this->payload_length() == 0); } + void calculate_payload_offset(); + Span ip_data() { return {ip_data_ptr(), ip_data_length()}; } @@ -185,17 +186,18 @@ namespace net { return payload(); } - const Byte* ip_data_ptr() const noexcept __attribute__((assume_aligned(4))) { return payload(); } private: + ip6::Header& ip6_header() noexcept + { return *reinterpret_cast(layer_begin()); } const ip6::Header& ip6_header() const noexcept { return *reinterpret_cast(layer_begin()); } - }; //< class PacketIP6 + }; //< PacketIP6 } //< namespace net #endif diff --git a/src/net/ip6/icmp6.cpp b/src/net/ip6/icmp6.cpp index 7a41fe20bd..a29e52cb94 100644 --- a/src/net/ip6/icmp6.cpp +++ b/src/net/ip6/icmp6.cpp @@ -102,6 +102,7 @@ namespace net case (ICMP_type::ND_NEIGHBOUR_SOL): case (ICMP_type::ND_NEIGHBOUR_ADV): case (ICMP_type::ND_REDIRECT): + PRINT(" NDP message from %s\n", req.ip().ip_src().str().c_str()); ndp().receive(req); break; case (ICMP_type::ROUTER_RENUMBERING): @@ -120,9 +121,8 @@ namespace net // The icmp6::Packet's payload contains the original packet sent that resulted // in an error - int payload_idx = req.payload_index(); auto packet_ptr = req.release(); - packet_ptr->increment_layer_begin(payload_idx); + packet_ptr->increment_layer_begin(req.payload_index()); // inet forwards to transport layer (UDP or TCP) inet_.error_report(err, std::move(packet_ptr)); @@ -135,9 +135,8 @@ namespace net // The icmp6::Packet's payload contains the original packet sent that resulted // in the Fragmentation Needed - int payload_idx = req.payload_index(); auto packet_ptr = req.release(); - packet_ptr->increment_layer_begin(payload_idx); + packet_ptr->increment_layer_begin(req.payload_index()); // Inet updates the corresponding Path MTU value in IP and notifies the transport/packetization layer inet_.error_report(err, std::move(packet_ptr)); @@ -232,8 +231,6 @@ namespace net PRINT(" Transmitting request to %s\n", dest_ip.to_string().c_str()); - printf(" Request size: %i payload size: %i\n", - req.ip().size(), req.payload().size()); // Default payload req.add_payload(includeos_payload.data(), includeos_payload.size()); diff --git a/src/net/ip6/ip6.cpp b/src/net/ip6/ip6.cpp index c261906327..b73e75d8d5 100644 --- a/src/net/ip6/ip6.cpp +++ b/src/net/ip6/ip6.cpp @@ -94,7 +94,6 @@ namespace net { auto reader = packet.layer_begin() + IP6_HEADER_LEN; auto next_proto = packet.next_protocol(); - uint8_t ext_len; uint16_t pl_off = IP6_HEADER_LEN; while (next_proto != Protocol::IPv6_NONXT) { @@ -103,23 +102,46 @@ namespace net } else if (next_proto == Protocol::OPTSV6) { } else { PRINT("Done parsing extension header, next proto: %d\n", next_proto); + packet.set_payload_offset(pl_off); return next_proto; } auto& ext = *(ip6::ExtensionHeader*)reader; - ext_len = ext.size(); - reader += ext_len; - //packet.update_extension_header_len(ext_len); - pl_off += ext_len; + auto ext_len = ext.size(); next_proto = ext.next(); + pl_off += ext_len; + reader += ext_len; } packet.set_payload_offset(pl_off); return next_proto; } + void PacketIP6::calculate_payload_offset() + { + auto reader = this->layer_begin() + IP6_HEADER_LEN; + auto next_proto = this->next_protocol(); + uint16_t pl_off = IP6_HEADER_LEN; + + while (next_proto != Protocol::IPv6_NONXT) + { + if (next_proto != Protocol::HOPOPT && + next_proto != Protocol::OPTSV6) + { + PRINT("Done parsing extension header, next proto: %d\n", next_proto); + this->set_payload_offset(pl_off); + return; + } + auto& ext = *(ip6::ExtensionHeader*)reader; + next_proto = ext.next(); + pl_off += ext.size(); + reader += ext.size(); + } + this->set_payload_offset(pl_off); + } void IP6::receive(Packet_ptr pckt, const bool /*link_bcast */) { - // Cast to IP6 Packet auto packet = static_unique_ptr_cast(std::move(pckt)); + // this will calculate exthdr length and set payload correctly + packet->calculate_payload_offset(); PRINT(" Source IP: %s, Dest.IP: %s, Next header: %d," "Payload len: %u, Hop limit: %d, version: %d, tc: %u, fl: %u\n", diff --git a/test/net/integration/tcp/service.cpp b/test/net/integration/tcp/service.cpp index b377dbe391..2b7ae102a8 100644 --- a/test/net/integration/tcp/service.cpp +++ b/test/net/integration/tcp/service.cpp @@ -157,7 +157,7 @@ void Service::start() { 8, 8, 8, 8 } // DNS ); inet.network_config6( - { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x93bd }, // IP6 + { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x85bd }, // IP6 64, // Prefix6 { 0xfe80, 0, 0, 0, 0xe823, 0xfcff, 0xfef4, 0x83e7 } // Gateway6 ); From 66474577dda9389b91737a1fa9217e9db8dec44d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 22 Jun 2018 17:09:29 +0200 Subject: [PATCH 442/723] cmake: Fix libgcc detection for GNU C --- cmake/post.service.cmake | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index ce866b2816..f70d239354 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -326,8 +326,11 @@ set_target_properties(libpthread PROPERTIES IMPORTED_LOCATION "${INSTALL_LOC}/${ # libgcc/compiler-rt detection if (UNIX) + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") + set(TARGET_LINE --target=${TRIPLE}) + endif() execute_process( - COMMAND ${CMAKE_CXX_COMPILER} --target=${TRIPLE} --print-libgcc-file-name + COMMAND ${CMAKE_CXX_COMPILER} ${TARGET_LINE} --print-libgcc-file-name RESULT_VARIABLE CC_RT_RES OUTPUT_VARIABLE COMPILER_RT_FILE OUTPUT_STRIP_TRAILING_WHITESPACE) if (NOT ${CC_RT_RES} EQUAL 0) From 09afb31918973b03da964bc91932439a7551a3cb Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 22 Jun 2018 17:10:34 +0200 Subject: [PATCH 443/723] cmos: Fix overflow/broken to_string() --- src/platform/x86_pc/cmos.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/platform/x86_pc/cmos.cpp b/src/platform/x86_pc/cmos.cpp index edd4b284d0..fcf7eb4d75 100644 --- a/src/platform/x86_pc/cmos.cpp +++ b/src/platform/x86_pc/cmos.cpp @@ -60,12 +60,14 @@ CMOS::Time& CMOS::Time::hw_update() { } -std::string CMOS::Time::to_string(){ - std::array str; - sprintf(str.data(), "%.2i-%.2i-%iT%.2i:%.2i:%.2iZ", +std::string CMOS::Time::to_string() +{ + std::array buffer; + int len = snprintf(buffer.data(), buffer.size(), + "%.2i-%.2i-%iT%.2i:%.2i:%.2iZ", f.year, f.month, f.day_of_month, f.hour, f.minute, f.second); - return std::string(str.data(), str.size()); + return std::string(buffer.data(), len); } From dfdac8708dae60598bebdda542f2a66b75597626 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 22 Jun 2018 17:11:32 +0200 Subject: [PATCH 444/723] liveupdate: Use assignment to zero headers --- lib/LiveUpdate/partition.cpp | 2 +- lib/LiveUpdate/storage.cpp | 7 ++++--- lib/LiveUpdate/storage.hpp | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/LiveUpdate/partition.cpp b/lib/LiveUpdate/partition.cpp index 2d5de6a7e2..b46ada2a66 100644 --- a/lib/LiveUpdate/partition.cpp +++ b/lib/LiveUpdate/partition.cpp @@ -75,7 +75,7 @@ void storage_header::zero_partition(int p) { auto& part = ptable.at(p); memset(&this->vla[part.offset], 0, part.length); - memset(&part, 0, sizeof(partition_header)); + part = {}; // NOTE: generate **NEW** checksum for header this->crc = generate_checksum(); } diff --git a/lib/LiveUpdate/storage.cpp b/lib/LiveUpdate/storage.cpp index c510e34834..b19a21d2ef 100644 --- a/lib/LiveUpdate/storage.cpp +++ b/lib/LiveUpdate/storage.cpp @@ -34,7 +34,7 @@ inline uint32_t liu_crc32(const void* buf, size_t len) const uint64_t storage_header::LIVEUPD_MAGIC = 0xbaadb33fdeadc0de; storage_header::storage_header() - : magic(LIVEUPD_MAGIC), crc(0), entries(0), length(0) + : magic(LIVEUPD_MAGIC) { //printf("%p --> %#llx\n", this, value); } @@ -174,8 +174,9 @@ void storage_header::try_zero() noexcept } void storage_header::zero() { - memset(this, 0, sizeof(storage_header) + this->length); - assert(this->magic == 0); + memset(this->vla, 0, this->length); + *this = {}; + this->magic = 0; } storage_entry* storage_header::begin(int p) diff --git a/lib/LiveUpdate/storage.hpp b/lib/LiveUpdate/storage.hpp index de822864bd..352c2f9ed4 100644 --- a/lib/LiveUpdate/storage.hpp +++ b/lib/LiveUpdate/storage.hpp @@ -153,7 +153,7 @@ struct storage_header void zero(); uint64_t magic; - mutable uint32_t crc; + mutable uint32_t crc = 0; uint32_t entries = 0; uint32_t length = 0; std::array ptable; From 874fcb7ee8375d8f5a54a597ad311720e6c3cd5f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 22 Jun 2018 17:12:16 +0200 Subject: [PATCH 445/723] linux: Make sure panic doesnt return --- linux/src/os.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/linux/src/os.cpp b/linux/src/os.cpp index 1645de87ec..e93c39206d 100644 --- a/linux/src/os.cpp +++ b/linux/src/os.cpp @@ -156,4 +156,5 @@ void panic(const char* why) { printf("!! PANIC !!\nReason: %s\n", why); raise(SIGINT); + exit(1); } From a4bf92242b76b712d8a51c277005e2b1dab08311 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 22 Jun 2018 17:12:45 +0200 Subject: [PATCH 446/723] seed: Add extra output for GNU C --- seed/service/service.cpp | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/seed/service/service.cpp b/seed/service/service.cpp index 4d197ac9a5..26a1f707c9 100644 --- a/seed/service/service.cpp +++ b/seed/service/service.cpp @@ -17,28 +17,14 @@ #include #include -#include -#include - -static void print_time() -{ - /* Obtain the time of day, and convert it to a tm struct. */ - struct timeval tv; - gettimeofday (&tv, NULL); - struct tm* ptm = localtime(&tv.tv_sec); - /* Format the date and time, down to a single second. */ - char time_string[40]; - strftime (time_string, sizeof (time_string), "%Y-%m-%d %H:%M:%S", ptm); - /* Compute milliseconds from microseconds. */ - long milliseconds = tv.tv_usec / 1000; - /* Print the formatted time, in seconds, followed by a decimal point - and the milliseconds. */ - printf ("%s.%03ld\n", time_string, milliseconds); -} +#include void Service::start(const std::string& args) { - printf("Hello world! Time is now "); print_time(); +#ifdef __GNUG__ + printf("Built by g++ " __VERSION__ "\n"); +#endif + printf("Hello world! Time is now %s\n", isotime::now().c_str()); printf("Args = %s\n", args.c_str()); printf("Try giving the service less memory, eg. 5MB in vm.json\n"); } From 69cc9e27cab4a918a1ed35bd2518d7d07b8368b0 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 22 Jun 2018 19:40:19 +0200 Subject: [PATCH 447/723] util: Fix wrong asserts in alloc_lstack --- api/util/alloc_lstack.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/util/alloc_lstack.hpp b/api/util/alloc_lstack.hpp index d6765fe023..0f78756938 100644 --- a/api/util/alloc_lstack.hpp +++ b/api/util/alloc_lstack.hpp @@ -127,8 +127,8 @@ class Lstack { private: Chunk* new_chunk(void* addr_begin, Chunk* next, size_t sz){ - Expects((sz & align - 1) == 0); - Expects(((uintptr_t)addr_begin & align - 1) == 0); + Expects((sz & (align - 1)) == 0); + Expects(((uintptr_t)addr_begin & (align - 1)) == 0); return new ((Chunk*)addr_begin) Chunk(next, sz); } From 41683e95c29d581c8d3f4f70e575c3cfce98df8f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 22 Jun 2018 19:40:41 +0200 Subject: [PATCH 448/723] musl: Fix unused warning for strace --- src/musl/common.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/musl/common.hpp b/src/musl/common.hpp index 532873c0ad..f6e2d78509 100644 --- a/src/musl/common.hpp +++ b/src/musl/common.hpp @@ -71,6 +71,7 @@ inline auto strace(Fn func, const char* name, Args&&... args) { if constexpr (__strace) strace_print(name, ret, args...); + (void) name; return ret; } From 2c55cfada0b509a58ea54200bb9b2f3b98d504a3 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 22 Jun 2018 19:41:59 +0200 Subject: [PATCH 449/723] x2apic: Work-around for g++ build --- src/platform/x86_pc/x2apic.hpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/platform/x86_pc/x2apic.hpp b/src/platform/x86_pc/x2apic.hpp index 0c08b04380..dbb97dba21 100644 --- a/src/platform/x86_pc/x2apic.hpp +++ b/src/platform/x86_pc/x2apic.hpp @@ -57,10 +57,11 @@ namespace x86 { static const uint8_t SPURIOUS_INTR = IRQ_BASE + Events::NUM_EVENTS-1; x2apic() { - INFO("x2APIC", "Enabling x2APIC"); - // add x2APIC enable bit to APIC BASE MSR auto base_msr = CPU::read_msr(IA32_APIC_BASE_MSR); - base_msr = (base_msr & 0xfffff100) | MSR_ENABLE_X2APIC; + INFO("x2APIC", "Enabling x2APIC @ %#x", (uint32_t) base_msr); + + // add x2APIC enable bit to APIC BASE MSR + base_msr = (base_msr & 0xfffff000) | MSR_ENABLE_X2APIC; // turn the x2APIC on CPU::write_msr(IA32_APIC_BASE_MSR, base_msr, 0); // verify that x2APIC is online @@ -219,7 +220,8 @@ namespace x86 { } uint32_t timer_diff() noexcept override { - return read(x2APIC_TMRINITCNT) - read(x2APIC_TMRCURRCNT); + return CPU::read_msr(BASE_MSR + x2APIC_TMRINITCNT) - + CPU::read_msr(BASE_MSR + x2APIC_TMRCURRCNT); } void timer_interrupt(bool enabled) noexcept override { From d226982a89589ed3a270c9e6a6ea2e96f61bb153 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 22 Jun 2018 21:11:12 +0200 Subject: [PATCH 450/723] x86: Dont make read/write virtual on APIC if --- src/platform/x86_pc/apic_iface.hpp | 3 --- src/platform/x86_pc/x2apic.hpp | 7 +++---- src/platform/x86_pc/xapic.hpp | 4 ++-- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/platform/x86_pc/apic_iface.hpp b/src/platform/x86_pc/apic_iface.hpp index 150595b360..ef3d3c5a19 100644 --- a/src/platform/x86_pc/apic_iface.hpp +++ b/src/platform/x86_pc/apic_iface.hpp @@ -26,9 +26,6 @@ namespace x86 { class IApic { public: - virtual uint32_t read (uint32_t reg) noexcept = 0; - virtual void write(uint32_t reg, uint32_t value) noexcept = 0; - virtual const char* name() const noexcept = 0; virtual uint32_t get_id() noexcept = 0; virtual uint32_t version() noexcept = 0; diff --git a/src/platform/x86_pc/x2apic.hpp b/src/platform/x86_pc/x2apic.hpp index dbb97dba21..70631b0054 100644 --- a/src/platform/x86_pc/x2apic.hpp +++ b/src/platform/x86_pc/x2apic.hpp @@ -70,7 +70,7 @@ namespace x86 { INFO2("APIC id: %x ver: %x", get_id(), version()); } - uint32_t read(uint32_t reg) noexcept override + uint32_t read(uint32_t reg) noexcept { return CPU::read_msr(BASE_MSR + reg); } @@ -78,7 +78,7 @@ namespace x86 { { return CPU::read_msr(BASE_MSR + reg); } - void write(uint32_t reg, uint32_t value) noexcept override + void write(uint32_t reg, uint32_t value) noexcept { CPU::write_msr(BASE_MSR + reg, value); } @@ -220,8 +220,7 @@ namespace x86 { } uint32_t timer_diff() noexcept override { - return CPU::read_msr(BASE_MSR + x2APIC_TMRINITCNT) - - CPU::read_msr(BASE_MSR + x2APIC_TMRCURRCNT); + return read(x2APIC_TMRINITCNT) - read(x2APIC_TMRCURRCNT); } void timer_interrupt(bool enabled) noexcept override { diff --git a/src/platform/x86_pc/xapic.hpp b/src/platform/x86_pc/xapic.hpp index 153a6acc71..a94a6e291c 100644 --- a/src/platform/x86_pc/xapic.hpp +++ b/src/platform/x86_pc/xapic.hpp @@ -78,11 +78,11 @@ namespace x86 { INFO2("ID: %x Ver: %x", get_id(), version()); } - uint32_t read(uint32_t reg) noexcept override + uint32_t read(uint32_t reg) noexcept { return *(volatile uint32_t*) (regbase + reg); } - void write(uint32_t reg, uint32_t value) noexcept override + void write(uint32_t reg, uint32_t value) noexcept { *(volatile uint32_t*) (regbase + reg) = value; } From 3a4c21956031171971dbe9c5207e84d12fdca9b6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 22 Jun 2018 21:12:10 +0200 Subject: [PATCH 451/723] x86/cpu: minor cleanup --- api/arch/x86/cpu.hpp | 4 +--- src/arch/x86_64/syscall_entry.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/api/arch/x86/cpu.hpp b/api/arch/x86/cpu.hpp index 7e8376b1f2..b6d0fd0289 100644 --- a/api/arch/x86/cpu.hpp +++ b/api/arch/x86/cpu.hpp @@ -21,7 +21,6 @@ #include #include -#include #define IA32_EFER 0xC0000080 #define IA32_STAR 0xC0000081 @@ -41,10 +40,9 @@ namespace x86 static uint64_t read_msr(uint32_t addr) { - #if defined(ARCH_x86) uint64_t v; - asm volatile("rdmsr": "=A" (v) : "c" (addr)); + asm volatile("rdmsr" : "=A" (v) : "c" (addr)); return v; #else #error "read_msr() not implemented for selected arch" diff --git a/src/arch/x86_64/syscall_entry.cpp b/src/arch/x86_64/syscall_entry.cpp index e8374b7fbc..6284ff1f41 100644 --- a/src/arch/x86_64/syscall_entry.cpp +++ b/src/arch/x86_64/syscall_entry.cpp @@ -15,14 +15,16 @@ // limitations under the License. #include -#include +#include +#include +#include +#include #define ARCH_SET_GS 0x1001 #define ARCH_SET_FS 0x1002 #define ARCH_GET_FS 0x1003 #define ARCH_GET_GS 0x1004 - #ifdef __x86_64__ static int sys_prctl(int code, uintptr_t ptr) { @@ -39,12 +41,10 @@ static int sys_prctl(int code, uintptr_t ptr) break; case ARCH_GET_GS: panic(" GET_GS called!\n"); - break; case ARCH_GET_FS: panic(" GET_FS called!\n"); - break; } - return 0; + return -EINVAL; } #endif From b6ccd508574c85a73f27d6fa0b9e90f69f7f8ade Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Sun, 24 Jun 2018 23:12:15 +0200 Subject: [PATCH 452/723] kernel: add buddy allocator as mmap backend --- api/util/alloc_buddy.hpp | 643 +++++++++++++++++++++++++++++++++++++++ api/util/bitops.hpp | 82 ++--- src/musl/mmap.cpp | 21 +- 3 files changed, 698 insertions(+), 48 deletions(-) create mode 100644 api/util/alloc_buddy.hpp diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp new file mode 100644 index 0000000000..14ec6d2ede --- /dev/null +++ b/api/util/alloc_buddy.hpp @@ -0,0 +1,643 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef UTIL_ALLOC_BUDDY_HPP +#define UTIL_ALLOC_BUDDY_HPP + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +// +// Tree node flags +// + +namespace mem::buddy { + enum class Flags : uint8_t { + free = 0, + taken = 1, + left_used = 2, + right_used = 4, + left_full = 8, + right_full = 16, + }; +} + +namespace util { + template<> + struct enable_bitmask_ops { + using type = uint8_t; + static constexpr bool enable = true; + }; + + template<> + struct enable_bitmask_ops { + using type = uint8_t; + static constexpr bool enable = true; + }; +} + +namespace mem::buddy { + using namespace util::literals; + using namespace util::bitops; + + using Node_t = uint8_t; + using Addr_t = uintptr_t; // Use void* only at outermost api level + using Size_t = size_t; + using Node_arr = gsl::span; + using Index_t = Node_arr::index_type; + + /** + * A buddy allocator over a fixed size pool + **/ + template + struct Alloc { + using Size_t = Size_t; + using Addr_t = Addr_t; + + static constexpr Size_t min_size = 4096; + static constexpr Size_t align = min_size; + + static constexpr Index_t node_count(Size_t pool_size){ + return ((pool_size / min_size) * 2) - 1; + } + + + + /** Total allocatable memory in an allocator created in a bufsize buffer **/ + static Size_t pool_size(Size_t bufsize){ + using namespace util; + + // Find closest power of two below bufsize + auto pool_size_ = bits::keeplast(bufsize - 1); + auto unalloc = bufsize - pool_size_; + auto node_size = node_count(pool_size_) * sizeof(Node_t); + auto overhead = bits::roundto(min_size, sizeof(Alloc) + node_size); + + // If bufsize == overhead + pow2, and overhead was too small to fit alloc + // Try the next power of two recursively + if(unalloc < overhead) + return pool_size(pool_size_); + + auto free = bufsize - overhead; + auto pool_size = bits::keeplast(free); + return pool_size; + } + + + /** Required total bufsize to manage pool_size memory **/ + static Size_t required_size(Size_t pool_size) { + using namespace util; + return bits::roundto(min_size, pool_size) + + bits::roundto(min_size, sizeof(Alloc) + node_count(pool_size) * sizeof(Node_t)); + } + + Index_t node_count() const noexcept { + return node_count(pool_size_); + } + + int leaf0() const noexcept { + return node_count() / 2 - 1; + } + + int tree_height() const noexcept { + return util::bits::fls(node_count()); + } + + int tree_width() const noexcept { + return node_count() / 2 + 1; + } + + int max_chunksize() const noexcept { + // Same as pool size. + return min_size << (tree_height() - 1); + } + + Size_t pool_size() + { return pool_size_; } + + + Size_t bytes_used() { + return bytes_used_; + } + + Addr_t highest_used() { + return root().highest_used(); + } + + Size_t bytes_free(){ + return pool_size_ - bytes_used_; + } + + Size_t bytes_free_r(){ + return pool_size_ - root().bytes_used(); + } + + Size_t bytes_used_r() { + return root().bytes_used_r(); + } + + bool full() { + return root().is_full(); + } + + bool empty() { + return root().is_free(); + } + + Addr_t addr_begin() { + return reinterpret_cast(start_addr_); + } + + Addr_t addr_end() { + return start_addr_ + pool_size_; + } + + bool in_range(void* a) { + auto addr = reinterpret_cast(a); + return addr >= addr_begin() and addr < addr_end(); + } + + static_assert(util::bits::is_pow2(min_size), "Page size must be power of 2"); + + + Alloc(void* start, Size_t bufsize, Size_t pool_size) + : nodes_{(Node_t*)start, node_count(pool_size)}, + start_addr_{util::bits::roundto(min_size, (Addr_t)start + node_count(pool_size))}, + pool_size_{pool_size} + + { + kprint("Constructing allocator \n"); + using namespace util; + Expects(bits::is_pow2(pool_size_)); + Expects(pool_size_ >= min_size); + Expects(bits::is_aligned(start)); + Expects(bits::is_aligned(start_addr_)); + + Ensures(pool_size_ + nodes_.size() * sizeof(Node_t) <= bufsize); + kprintf("Constructed alloc. Nodes start 0x%x, node count 0x%x ", + nodes_.data(), nodes_.size()); + // Initialize nodes + memset(nodes_.data(), 0, nodes_.size() * sizeof(Node_arr::element_type)); + } + + static Alloc* create(void* addr, Size_t bufsize) { + using namespace util; + Size_t pool_size_ = pool_size(bufsize); + Size_t overhead_ = bufsize - pool_size_; + auto* nodes_ptr = (char*)addr + overhead_; + kprintf("Creating buddy allocator, bufsize 0x%zx, pool_size_ 0x%zx required_size 0x%zx " + "overhead: 0x%zx \n", + bufsize, pool_size_, required_size(pool_size_), overhead_); + + Expects(bufsize >= required_size(pool_size_)); + Ensures(bufsize >= required_size(pool_size_)); + auto* alloc = new (addr) Alloc(nodes_ptr, bufsize, pool_size_); + return alloc; + } + + + Size_t chunksize(Size_t wanted_sz) { + auto sz = util::bits::next_pow2(wanted_sz); + if (sz > max_chunksize()) + return 0; + return std::max(sz, min_size); + } + + enum class Track { + get = 0, + inc = 1, + start = 2 + }; + + struct Track_res { + int last; + int total; + int min; + int max; + int allocs; + }; + + Track_res alloc_tracker(Track action = Track::get) { + if constexpr (Track_allocs) { + static Track_res tr {0,0,0,0,0}; + if (action == Track::start) { + + tr.allocs++; + + if (tr.last < tr.min or tr.min == 0) + tr.min = tr.last; + + if (tr.last > tr.max) + tr.max = tr.last; + + tr.total += tr.last; + tr.last = 0; + + } else if (action == Track::inc){ + tr.last++; + } + return tr; + } + return {}; + } + + void* allocate(Size_t size) { + + Expects(start_addr_); + + auto node = root(); + auto addr = start_addr_; + + auto sz = chunksize(size); + if (not sz) return 0; + + // Start allocation tracker + alloc_tracker(Track::start); + + auto res = node.allocate(sz); + if (res) bytes_used_ += sz; + return reinterpret_cast(res); + } + + void deallocate(void* addr, Size_t size) { + auto sz = size ? chunksize(size) : 0; + auto res = root().deallocate((Addr_t)addr, sz); + Expects(not size or res == sz); + bytes_used_ -= res; + } + + void free(void* addr) { + deallocate(addr, 0); + } + + + //private: + class Node_view { + public: + + Node_view(int index, Alloc* alloc) + : i_{index}, alloc_{alloc}, + my_size_{compute_size()}, my_addr_{compute_addr()} + {} + + Node_view() = default; + + Node_view left() { + if (is_leaf()) { + return Node_view(); + } + return Node_view(i_ * 2 + 1, alloc_); + } + + Node_view right() { + if (is_leaf()) { + return Node_view(); + } + return Node_view(i_ * 2 + 2, alloc_); + } + + Addr_t addr() + { return my_addr_; } + + Size_t size() + { return my_size_; } + + bool is_leaf() { + return i_ >= alloc_->nodes_.size() / 2; + } + + bool is_taken() { + return data() & Flags::taken; + } + + bool is_free() { + return data() == 0; + } + + bool is_full() { + auto fl = data(); + return fl & Flags::taken or + (fl & Flags::left_full and fl & Flags::right_full); + } + + bool in_range(Addr_t a) { + return a >= my_addr_ and a < my_addr_ + my_size_; + } + + void set_flag(Flags f){ + data() |= f; + } + + void clear_flag(Flags f){ + data() &= ~f; + } + + bool is_parent() { + return i_ <= alloc_->leaf0(); + } + + + bool is_full_r() { + if (is_taken()) + return true; + return is_parent() and left().is_full_r() + and is_parent() and right().is_full_r(); + } + + bool is_free_r() { + auto lfree = not is_parent() or left().is_free_r(); + auto rfree = not is_parent() or right().is_free_r(); + return not is_taken() and lfree and rfree; + } + + + int height() { + return (util::bits::fls(i_ + 1)); + } + + Size_t compute_size() { + return min_size << (alloc_->tree_height() - height()); + } + + Addr_t compute_addr() { + auto first_lvl_idx = 1 << (height() - 1); + auto sz_offs = i_ - first_lvl_idx + 1; + return alloc_->start_addr_ + (sz_offs * my_size_); + } + + operator bool() { + return alloc_->start_addr_ != 0 and alloc_ != nullptr; + } + + Node_t& data() { + return alloc_->nodes_.at(i_); + } + + Addr_t allocate_self() { + Expects(not (data() & Flags::taken)); + set_flag(Flags::taken); + return my_addr_; + } + + Addr_t allocate(Size_t sz) { + using namespace util; + + // Track allocation steps + alloc_->alloc_tracker(Track::inc); + + Expects(sz <= my_size_); + Expects(! is_taken()); + + if (is_taken()) { + return 0; + } + + // Allocate self if possible + if (sz == my_size_) { + if (is_free()) { + return allocate_self(); + } + return 0; + } + + if (not is_parent()) + return 0; + + // Try left allocation + auto lhs = left(); + if (not lhs.is_full()) { + + auto addr_l = lhs.allocate(sz); + + if (lhs.is_full()) + set_flag(Flags::left_full); + + if (addr_l) { + set_flag(Flags::left_used); + return addr_l; + } + } + + // Try right allocation + auto rhs = right(); + if (not rhs.is_full()) { + + auto addr_r = rhs.allocate(sz); + + if (rhs.is_full()) + set_flag(Flags::right_full); + + if (addr_r != 0) { + set_flag(Flags::right_used); + return addr_r; + } + } + + return 0; + } + + // Left / right deallocation + // TODO: there must be a pattern to make this DRY in a nice way + // also for the left / right allocation steps above. + + Size_t dealloc_left(Addr_t ptr, Size_t sz) { + auto lhs = left(); + auto res = lhs.deallocate(ptr, sz); + if (not lhs.is_full()) { + clear_flag(Flags::left_full); + } + if (lhs.is_free()) { + clear_flag(Flags::left_used); + } + return res; + } + + Size_t dealloc_right(Addr_t ptr, Size_t sz) { + auto rhs = right(); + auto res = rhs.deallocate(ptr, sz); + if (not rhs.is_full()){ + clear_flag(Flags::right_full); + } + if (rhs.is_free()) { + clear_flag(Flags::right_used); + } + return res; + } + + Size_t dealloc_self(){ + data() = 0; + return my_size_; + } + + bool is_left(Addr_t ptr){ + return ptr < my_addr_ + my_size_ / 2; + } + + bool is_right(Addr_t ptr) { + return ptr >= my_addr_ + my_size_ / 2; + } + + Addr_t deallocate(Addr_t ptr, Size_t sz = 0) { + using namespace util; + + #ifdef DEBUG_UNIT + printf("Node %i DE-allocating 0x%zx size %zi \n", i_, ptr, sz); + #endif + + Expects(not (is_taken() and (data() & Flags::left_used + or data() & Flags::right_used))); + + + // Don't try if ptr is outside range + if (not in_range(ptr)) { + return 0; + } + + if (is_taken()) { + if (ptr != my_addr_) + return 0; + + if (sz) + Expects(my_size_ == sz); + + return dealloc_self(); + } + + if (not is_parent()) { + return 0; + } + + if (is_left(ptr)) { + return dealloc_left(ptr, sz); + } + + if (is_right(ptr)) { + return dealloc_right(ptr, sz); + } + + return 0; + } + + Size_t bytes_used() { + if (is_taken()) + return my_size_; + + if (is_parent()) + return left().bytes_used() + right().bytes_used(); + + return 0; + } + + Addr_t highest_used() { + if (is_taken() or not is_parent()) + return my_addr_ + my_size_; + + auto rhs = right().highest_used(); + if (rhs > my_addr_ + my_size_) return rhs; + return left().highest_used(); + } + + std::string to_string(){ + std::stringstream out; + out << std::hex + // << addr()<< " | " + << (int)data() //is_full_r() // + << std::dec; + return out.str(); + } + + private: + int i_ = 0; + Alloc* alloc_ = nullptr; + Size_t my_size_ = 0; + Addr_t my_addr_ = 0; + }; + + Node_view root() + { return Node_view(0, this); } + + std::string summary() { + std::stringstream out; + std::string dashes(80, '-'); + out << dashes << "\n"; + out << "Bytes used: " << util::Byte_r(bytes_used()) + << " Bytes free: " << util::Byte_r(bytes_free()) + << " H: " << std::dec << tree_height() + << " W: " << tree_width() + << " Alloc.size: " << util::Byte_r(sizeof(*this)) << "\n" + << "Address pool: 0x" << std::hex + << root().addr() << " - " << root().addr() + pool_size_ + << std::dec << "\n"; + auto track = alloc_tracker(); + out << "Allocations: " << track.allocs + << " Steps: Last: " << track.last + << " Min: " << track.min + << " Max: " << track.max + << " Total: " << track.total + << " Avg: " << track.total / track.allocs + << "\n"; + out << dashes << "\n"; + return out.str(); + } + std::string draw_tree(bool index = false) { + auto node_pr_w = index ? 6 : 4; + auto line_w = tree_width() * node_pr_w; + + std::stringstream out{summary()}; + std::string spaces (line_w, ' '); + std::string dashes (line_w, '-'); + + int i = 0; + + for (int h = 0; h < tree_height(); h++) { + auto cnt = 1 << h; + auto prw = cnt * node_pr_w; + int loff = (spaces.size() / 2) - (prw / 2); + // Line offset + auto nsize = min_size << (tree_height() - h - 1); + printf("%s\t", util::Byte_r(nsize).to_string().c_str()); + printf("%.*s", loff, spaces.c_str()); + out.flush(); + while (cnt--) { + printf("[%s] ", Node_view(i, this).to_string().c_str()); + i++; + } + printf("\n"); + } + out << dashes << "\n"; + return out.str(); + } + + //using Tracker = typename std::enable_if::type; + Track_res tr {0,0,0,0,0}; // If tracking enabled + + Node_arr nodes_; + const uintptr_t start_addr_ = 0; + const Size_t pool_size_ = min_size; + Size_t bytes_used_ = 0; + }; + +} + +#endif diff --git a/api/util/bitops.hpp b/api/util/bitops.hpp index 041e42b27b..ca9f352145 100644 --- a/api/util/bitops.hpp +++ b/api/util/bitops.hpp @@ -113,17 +113,58 @@ namespace bits { // // Various bit operations // +// Number of bits per word +constexpr int bitcnt() +{ return sizeof(uintptr_t) * 8; } + +// Count leading zeroes +inline uintptr_t clz(uintptr_t n){ + return __builtin_clzl(n); +} + +// Count trailing zeroes +inline uintptr_t ctz(uintptr_t n){ + return __builtin_ctzl(n); +} + +// Find first bit set. +inline uintptr_t ffs(uintptr_t n){ + return __builtin_ffsl(n); +} + +// Find last bit set. The integral part of log2(n). +inline uintptr_t fls(uintptr_t n){ + return (uintptr_t) bitcnt() - clz(n); +} + +// Keep last set bit, clear the rest. +inline uintptr_t keeplast(uintptr_t n) +{ return 1LU << (fls(n) - 1); } + +// Keep first set bit, clear the rest. +inline uintptr_t keepfirst(uintptr_t n) +{ return 1LU << (ffs(n) - 1); } + +// Number of bits set in n +inline uintptr_t popcount(uintptr_t n) +{ return __builtin_popcountl(n); } // Check for power of two -inline bool is_pow2(uintptr_t i) +inline constexpr bool is_pow2(uintptr_t i) { return i && !(i & (i - 1)); } +inline uintptr_t next_pow2(uintptr_t i) +{ + auto lastbit = keeplast(i); + return i == lastbit ? i : lastbit * 2; +} + // Multiples of M required to cover x template inline uintptr_t multip(uintptr_t x) { return (M - 1 + x) / M; } -inline uintptr_t multip(uintptr_t M, uintptr_t x) +inline constexpr uintptr_t multip(uintptr_t M, uintptr_t x) {return (M - 1 + x) / M; } // Round up to nearest multiple of M @@ -131,7 +172,7 @@ template inline uintptr_t roundto(uintptr_t x) { return multip(x) * M; } -inline uintptr_t roundto(uintptr_t M, uintptr_t x) +inline constexpr uintptr_t roundto(uintptr_t M, uintptr_t x) { return multip(M,x) * M; } // Determine if ptr is A-aligned @@ -152,41 +193,6 @@ inline bool is_aligned(uintptr_t A, uintptr_t ptr) return (ptr & (A - 1)) == 0; } -// Number of bits per word -constexpr int bitcnt() -{ return sizeof(uintptr_t) * 8; } - -// Count leading zeroes -inline uintptr_t clz(uintptr_t n){ - return __builtin_clzl(n); -} - -// Count trailing zeroes -inline uintptr_t ctz(uintptr_t n){ - return __builtin_ctzl(n); -} - -// Find first bit set. -inline uintptr_t ffs(uintptr_t n){ - return __builtin_ffsl(n); -} - -// Find last bit set. The integral part of log2(n). -inline uintptr_t fls(uintptr_t n){ - return (uintptr_t) bitcnt() - clz(n); -} - -// Keep last set bit, clear the rest. -inline uintptr_t keeplast(uintptr_t n) -{ return 1LU << (fls(n) - 1); } - -// Keep first set bit, clear the rest. -inline uintptr_t keepfirst(uintptr_t n) -{ return 1LU << (ffs(n) - 1); } - -// Number of bits set in n -inline uintptr_t popcount(uintptr_t n) -{ return __builtin_popcountl(n); } } // ns bitops diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 694d194333..4f0850a18b 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -2,19 +2,20 @@ #include #include #include -#include +#include #include #include -using Alloc = util::alloc::Lstack<4096>; -static Alloc alloc; +using Alloc = mem::buddy::Alloc<>; +static Alloc* alloc; uintptr_t __init_mmap(uintptr_t addr_begin) { - Expects(alloc.empty()); auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); int64_t len = (OS::heap_max() - aligned_begin) & ~int64_t(Alloc::align - 1); - alloc.donate((void*)aligned_begin, len); + + kprintf("Initializing mmap\n"); + alloc = Alloc::create((void*)aligned_begin, len); kprintf("* mmap initialized. Begin: 0x%zx, end: 0x%zx\n", addr_begin, addr_begin + len); return aligned_begin + len; @@ -22,24 +23,24 @@ uintptr_t __init_mmap(uintptr_t addr_begin) extern "C" __attribute__((weak)) void* kalloc(size_t size) { - return alloc.allocate(size); + return alloc->allocate(size); } extern "C" __attribute__((weak)) void kfree (void* ptr, size_t size) { - alloc.deallocate(ptr, size); + alloc->deallocate(ptr, size); } size_t mmap_bytes_used() { - return alloc.bytes_allocated(); + return alloc->bytes_used(); } size_t mmap_bytes_free() { - return alloc.bytes_free(); + return alloc->bytes_free(); } uintptr_t mmap_allocation_end(){ - return alloc.allocation_end(); + return alloc->highest_used(); } static void* sys_mmap(void *addr, size_t length, int /*prot*/, int /*flags*/, From b1d7839e68f4fb3445961b7b365f52ffbb690ad3 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Sun, 24 Jun 2018 23:14:28 +0200 Subject: [PATCH 453/723] test: buddy allocator --- test/CMakeLists.txt | 1 + test/lest_util/common.cxx | 6 +- test/util/unit/buddy.cpp | 302 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 307 insertions(+), 2 deletions(-) create mode 100644 test/util/unit/buddy.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c0e5cc3d04..11d93b3b62 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -144,6 +144,7 @@ set(TEST_SOURCES ${TEST}/util/unit/uri_test.cpp ${TEST}/util/unit/bitops.cpp ${TEST}/util/unit/lstack.cpp + ${TEST}/util/unit/buddy.cpp ) set(OS_SOURCES diff --git a/test/lest_util/common.cxx b/test/lest_util/common.cxx index 0b5f3a7836..d35999e955 100644 --- a/test/lest_util/common.cxx +++ b/test/lest_util/common.cxx @@ -5,7 +5,8 @@ #define CASE( name ) lest_CASE( specification(), name ) extern lest::tests & specification(); -#define DEBUG_UNIT +#define TEST + #ifdef DEBUG_UNIT #define LINK printf("%s:%i: OK\n",__FILE__,__LINE__) #else @@ -31,7 +32,8 @@ static std::vector rands64() return rnd; } -const std::vector random = rands64<10>(); +const std::vector random = rands64<100>(); +const std::vector random_1k = rands64<1000>(); } diff --git a/test/util/unit/buddy.cpp b/test/util/unit/buddy.cpp new file mode 100644 index 0000000000..928ca67b0d --- /dev/null +++ b/test/util/unit/buddy.cpp @@ -0,0 +1,302 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#define DEBUG_UNIT + +#include +#include +#include + + + +struct Pool { + using Alloc = mem::buddy::Alloc; + + Pool(size_t s) : size{s} { + auto sz = Alloc::required_size(s); + printf("Pool: wanted size 0x%zx, required 0x%zx\n", s, sz); + auto res = posix_memalign(&addr, Alloc::min_size, sz); + Expects(res == 0); + alloc = Alloc::create(addr, sz); + } + + ~Pool() { + free((void*)addr); + } + + size_t size; + Alloc* alloc = nullptr; + void* addr = nullptr; +}; + +CASE("mem::buddy init allocator"){ + using namespace util; + Pool pool(64_KiB); + auto& alloc = *pool.alloc; + + #ifdef DEBUG_UNIT + std::cout << "Allocator root @ 0x" + << std::hex << alloc.root().addr() + << " size: " + << std::dec << alloc.root().size() + << "\n"; + #endif + + EXPECT(bool(alloc.root())); + + EXPECT(alloc.root().height() == 1); + + EXPECT(alloc.root().is_parent()); + EXPECT(alloc.root().is_free()); + EXPECT(alloc.root().left().height() == 2); + EXPECT(alloc.bytes_used() == 0); + auto addr = alloc.allocate(4_KiB); + EXPECT(addr); + EXPECT(alloc.bytes_used() == 4_KiB); + alloc.deallocate(addr, 4_KiB); + EXPECT(alloc.bytes_used() == 0); +} + +CASE("mem::buddy basic allocation / deallocation"){ + using namespace util; + + Pool pool(64_KiB); + auto& alloc = *pool.alloc; + auto node = alloc.root(); + + // Verify heights, leftmost addresses etc. + for (int i = 1; i < alloc.tree_height(); i++) { + EXPECT(node.height() == i); + if (node.is_parent()) { + EXPECT(node.left().height() == i + 1); + EXPECT(node.right().height() == i + 1); + } else { + EXPECT(node.left().height() == 0); + } + EXPECT(node.addr() == alloc.addr_begin()); + node = node.left(); + } + + EXPECT(alloc.root().addr() == alloc.addr_begin()); + EXPECT(not alloc.root().deallocate(0x10, 0x10000)); + + auto sum = 0; + std::vector addresses; + + // Allocate all allowable sizes + for (auto sz = alloc.min_size; sz < pool.size; sz *= 2) { + auto addr = alloc.allocate(sz); + EXPECT(addr); + EXPECT(alloc.in_range(addr)); + addresses.push_back(addr); + sum += sz; + EXPECT(alloc.bytes_used() == sum); + + #ifdef DEBUG_UNIT + std::cout << alloc.draw_tree(); + #endif + } + + // Deallocate + auto sz = alloc.min_size; + for (auto addr : addresses) { + sum -= sz; + alloc.free((void*)addr); + EXPECT(alloc.bytes_used() == sum); + sz *= 2; + } + + EXPECT(alloc.bytes_used() == 0); + +} + + +CASE("mem::buddy random ordered allocation then deallocation"){ + using namespace util; + #ifdef DEBUG_UNIT + std::cout << "mem::buddy random ordered\n"; + #endif + + Pool pool(32_MiB); + auto& alloc = *pool.alloc; + + EXPECT(bool(alloc.root())); + + auto sum = 0; + std::vector addresses; + std::vector sizes; + + // Allocate all allowable sizes + for (auto rnd : test::random_1k) { + + if (not alloc.bytes_free()) + break; + + const auto sz = alloc.chunksize(rnd % std::max(size_t(32_KiB), pool.size / 1024)); + + #ifdef DEBUG_UNIT + std::cout << "Alloc " << Byte_r(sz) << "\n"; + #endif + auto avail = alloc.bytes_free(); + if (sz > avail or sz == 0) { + if (alloc.full()) + break; + continue; + } + auto addr = alloc.allocate(sz); + EXPECT(addr); + EXPECT(alloc.in_range(addr)); + addresses.push_back(addr); + sizes.push_back(sz); + sum += sz; + EXPECT(alloc.bytes_used() == sum); + #ifdef DEBUG_UNIT + std::cout << alloc.summary(); + #endif + } + + auto dashes = std::string(80, '='); + + #ifdef DEBUG_UNIT + std::cout << "\nAlloc done. Now dealloc \n" << dashes << "\n"; + if (pool.size <= 256_KiB) + std::cout << alloc.draw_tree(); + #endif + + // Deallocate + for (int i = 0; i < addresses.size(); i++) { + auto addr = addresses.at(i); + auto sz = sizes.at(i); + if (addr) EXPECT(sz) ; + sum -= sz; + alloc.free((void*)addr); + EXPECT(alloc.bytes_used() == sum); + } + + #ifdef DEBUG_UNIT + std::cout << "Completed " << addresses.size() << " random ordered\n"; + std::cout << alloc.summary(); + #endif + + EXPECT(alloc.bytes_used() == 0); +} + +struct Allocation { + using Alloc = Pool::Alloc; + using Addr_t = typename Alloc::Addr_t; + using Size_t = typename Alloc::Size_t; + + Addr_t addr = 0; + Size_t size = 0; + char data = 0; + Alloc* alloc = nullptr; + + Allocation(Alloc* a) : alloc{a} {} + + auto addr_begin(){ + return addr; + } + + auto addr_end() { + return addr + size; + } + + bool overlaps(Addr_t other) { + return other >= addr_begin() and other < addr_end(); + } + + bool overlaps(Allocation other){ + return overlaps(other.addr_begin()) + or overlaps(other.addr_end()); + } + + bool verify_addr() { + return alloc->in_range((void*)addr); + } + + bool verify_all() { + if (not verify_addr()) + return false; + + auto buf = std::make_unique(size); + memset(buf.get(), data, size); + if (memcmp(buf.get(), (void*)addr, size) == 0) + return true; + return false; + } +}; + + +CASE("mem::buddy random chaos with data verification"){ + using namespace util; + + Pool pool(1_GiB); + auto& alloc = *pool.alloc; + + EXPECT(bool(alloc.root())); + EXPECT(alloc.bytes_free() == alloc.pool_size()); + + std::vector allocs; + + for (auto rnd : test::random_1k) { + auto sz = rnd % alloc.pool_size_ / 1024; + EXPECT(sz); + + if (not alloc.full()) { + Allocation a{&alloc}; + a.size = sz; + a.addr = (uintptr_t)alloc.allocate(sz); + a.data = 'A' + (rnd % ('Z' - 'A')); + EXPECT(a.addr); + EXPECT(a.verify_addr()); + EXPECT(a.overlaps(a)); + auto overlap = + std::find_if(allocs.begin(), allocs.end(), [&a](Allocation& other) { + return other.overlaps(a); + }); + EXPECT(overlap == allocs.end()); + allocs.emplace_back(std::move(a)); + memset((void*)a.addr, a.data, a.size); + } + + EXPECT(not alloc.empty()); + + // Delete a random allocation + if (rnd % 3 or alloc.full()) { + auto a = allocs.begin() + rnd % allocs.size(); + a->verify_all(); + auto use_pre = alloc.bytes_used(); + alloc.deallocate((void*)a->addr, a->size); + EXPECT(alloc.bytes_used() == use_pre - alloc.chunksize(a->size)); + allocs.erase(a); + } + } + + #ifdef DEBUG_UNIT + std::cout << "mem::buddy random chaos complete \n"; + std::cout << alloc.summary(); + #endif + + for (auto a : allocs) { + alloc.deallocate((void*)a.addr, a.size); + } + + #ifdef DEBUG_UNIT + std::cout << "mem::buddy random chaos cleaned up \n"; + #endif + + EXPECT(alloc.empty()); +} From d4e6fc7dfa1c4e73f459f8f11c027b2569e322b4 Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Mon, 25 Jun 2018 12:16:13 +0200 Subject: [PATCH 454/723] Updating NaCl (new dynamic implementation) --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index 874d77668f..7b7490c4aa 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit 874d77668f67e1b6bc3f003fbd969243a25bcb07 +Subproject commit 7b7490c4aa1597c832907aa934bcea0f25e29b3c From 3cac7fafe156c40da2cd5327d043e0af63a5003c Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Mon, 25 Jun 2018 14:15:48 +0200 Subject: [PATCH 455/723] nacl.cmake: Update to support new NaCl implementation --- cmake/nacl.cmake | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/cmake/nacl.cmake b/cmake/nacl.cmake index fa10b37ca7..0dff12393e 100644 --- a/cmake/nacl.cmake +++ b/cmake/nacl.cmake @@ -18,13 +18,27 @@ ExternalProject_Add(nacl_bin set(NACL_DIR ${INCLUDEOS_ROOT}/NaCl) set(NACL_EXE ${NACL_DIR}/NaCl.py) set(NACL_SRC - ${NACL_DIR}/cpp_transpile_function.py ${NACL_DIR}/cpp_template.mustache - ${NACL_DIR}/cpp_resolve_values.py - ${NACL_DIR}/shared_constants.py + ${NACL_DIR}/shared.py + ) +set(NACL_SUBTRANS_SRC + ${NACL_DIR}/subtranspilers/__init__.py + ${NACL_DIR}/subtranspilers/function_transpiler.py + ${NACL_DIR}/subtranspilers/value_transpiler.py + ) +set(NACL_TYPEPROC_SRC + ${NACL_DIR}/type_processors/__init__.py + ${NACL_DIR}/type_processors/conntrack.py + ${NACL_DIR}/type_processors/function.py + ${NACL_DIR}/type_processors/gateway.py + ${NACL_DIR}/type_processors/iface.py + ${NACL_DIR}/type_processors/load_balancer.py + ${NACL_DIR}/type_processors/syslog.py ) set(NACL_BIN ${CMAKE_CURRENT_BINARY_DIR}/nacl_bin/src/nacl_bin) install(PROGRAMS ${NACL_EXE} DESTINATION includeos/nacl) install(FILES ${NACL_SRC} DESTINATION includeos/nacl) +install(FILES ${NACL_SUBTRANS_SRC} DESTINATION includeos/nacl/subtranspilers) +install(FILES ${NACL_TYPEPROC_SRC} DESTINATION includeos/nacl/type_processors) install(DIRECTORY ${NACL_BIN}/ DESTINATION includeos/nacl) From a40eac5c814e9a6a96035b95f1905f9fe0c0019f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 25 Jun 2018 14:57:29 +0200 Subject: [PATCH 456/723] Fix build and test issues for Buddy Alloc --- api/util/alloc_buddy.hpp | 9 +++------ test/CMakeLists.txt | 2 -- test/lest_util/os_mock.cpp | 12 ++++++++++++ test/util/unit/buddy.cpp | 6 ++---- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index a021eca044..de5f71abab 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -72,13 +72,10 @@ namespace mem::buddy { **/ template struct Alloc { - using Size_t = Size_t; - using Addr_t = Addr_t; + static constexpr mem::buddy::Size_t min_size = 4096; + static constexpr mem::buddy::Size_t align = min_size; - static constexpr Size_t min_size = 4096; - static constexpr Size_t align = min_size; - - static constexpr Index_t node_count(Size_t pool_size){ + static constexpr Index_t node_count(mem::buddy::Size_t pool_size){ return ((pool_size / min_size) * 2) - 1; } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5ad241ef5f..cab4944065 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -182,13 +182,11 @@ set(OS_SOURCES ${SRC}/kernel/events.cpp ${SRC}/kernel/memmap.cpp ${SRC}/kernel/os.cpp - ${SRC}/kernel/heap.cpp ${SRC}/kernel/pci_manager.cpp ${SRC}/kernel/rng.cpp ${SRC}/kernel/service_stub.cpp ${SRC}/kernel/syscalls.cpp ${SRC}/kernel/timers.cpp - ${SRC}/musl/mmap.cpp ${SRC}/musl/brk.cpp ${SRC}/net/buffer_store.cpp ${SRC}/net/conntrack.cpp diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index 1c4855ce2b..9265a7face 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -213,4 +213,16 @@ bool rdrand32(uint32_t* result) { return true; } +/// heap /// +uintptr_t __brk_max = 0; +uintptr_t OS::heap_begin() noexcept { + return 0; +} +uintptr_t OS::heap_end() noexcept { + return 1 << 30; +} +uintptr_t OS::heap_max() noexcept { + return -1; +} + #endif diff --git a/test/util/unit/buddy.cpp b/test/util/unit/buddy.cpp index 847e09e8e7..2207f900c1 100644 --- a/test/util/unit/buddy.cpp +++ b/test/util/unit/buddy.cpp @@ -20,8 +20,6 @@ #include #include - - struct Pool { using Alloc = mem::buddy::Alloc; @@ -195,8 +193,8 @@ CASE("mem::buddy random ordered allocation then deallocation"){ struct Allocation { using Alloc = Pool::Alloc; - using Addr_t = typename Alloc::Addr_t; - using Size_t = typename Alloc::Size_t; + using Addr_t = mem::buddy::Addr_t; + using Size_t = mem::buddy::Size_t; Addr_t addr = 0; Size_t size = 0; From 10204b7b5fa3de7826f034a7d1a0cc0f3ee95e63 Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Mon, 25 Jun 2018 15:47:35 +0200 Subject: [PATCH 457/723] nacl.cmake: Copying the whole subtranspilers and type_processors directories instead of specifying file names --- cmake/nacl.cmake | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/cmake/nacl.cmake b/cmake/nacl.cmake index 0dff12393e..21f57494b1 100644 --- a/cmake/nacl.cmake +++ b/cmake/nacl.cmake @@ -21,24 +21,10 @@ set(NACL_SRC ${NACL_DIR}/cpp_template.mustache ${NACL_DIR}/shared.py ) -set(NACL_SUBTRANS_SRC - ${NACL_DIR}/subtranspilers/__init__.py - ${NACL_DIR}/subtranspilers/function_transpiler.py - ${NACL_DIR}/subtranspilers/value_transpiler.py - ) -set(NACL_TYPEPROC_SRC - ${NACL_DIR}/type_processors/__init__.py - ${NACL_DIR}/type_processors/conntrack.py - ${NACL_DIR}/type_processors/function.py - ${NACL_DIR}/type_processors/gateway.py - ${NACL_DIR}/type_processors/iface.py - ${NACL_DIR}/type_processors/load_balancer.py - ${NACL_DIR}/type_processors/syslog.py - ) set(NACL_BIN ${CMAKE_CURRENT_BINARY_DIR}/nacl_bin/src/nacl_bin) install(PROGRAMS ${NACL_EXE} DESTINATION includeos/nacl) install(FILES ${NACL_SRC} DESTINATION includeos/nacl) -install(FILES ${NACL_SUBTRANS_SRC} DESTINATION includeos/nacl/subtranspilers) -install(FILES ${NACL_TYPEPROC_SRC} DESTINATION includeos/nacl/type_processors) +install(DIRECTORY ${NACL_DIR}/subtranspilers DESTINATION includeos/nacl) +install(DIRECTORY ${NACL_DIR}/type_processors DESTINATION includeos/nacl) install(DIRECTORY ${NACL_BIN}/ DESTINATION includeos/nacl) From 2a6e289e52caaf469f80761f53635672d4a71b3d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 25 Jun 2018 17:00:55 +0200 Subject: [PATCH 458/723] net: Add redirection header for old inet4 for compatibility --- api/net/inet | 5 +++-- api/net/inet4 | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 api/net/inet4 diff --git a/api/net/inet b/api/net/inet index 5474bd2632..d2a9fbf949 100644 --- a/api/net/inet +++ b/api/net/inet @@ -1,6 +1,7 @@ //-*- C++ -*- -#ifndef NET_INET -#define NET_INET +#pragma once +#ifndef NET_INET_HEADER +#define NET_INET_HEADER #include "inet.hpp" diff --git a/api/net/inet4 b/api/net/inet4 new file mode 100644 index 0000000000..53e00a37fe --- /dev/null +++ b/api/net/inet4 @@ -0,0 +1,4 @@ +//-*- C++ -*- +#pragma once +#warning net/inet4 has changed to net/inet, please #include that instead! +#include "inet.hpp" From 655e3a58b81643dec29f08abf701efc46cb85070 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 26 Jun 2018 11:50:50 +0200 Subject: [PATCH 459/723] tcp: Dont call on_close on TCP stream if not set --- api/net/tcp/stream.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/net/tcp/stream.hpp b/api/net/tcp/stream.hpp index 0d068c5482..873cd79a22 100644 --- a/api/net/tcp/stream.hpp +++ b/api/net/tcp/stream.hpp @@ -103,7 +103,7 @@ namespace net::tcp auto onclose = std::move(this->m_on_close); m_tcp->reset_callbacks(); m_tcp->close(); - onclose(); + if (onclose) onclose(); } /** From 5c799625b64c506561c5b48ea84c67f4552a9020 Mon Sep 17 00:00:00 2001 From: Ingve Vormestrand Date: Tue, 26 Jun 2018 13:04:13 +0200 Subject: [PATCH 460/723] uplink: use UUID as id --- lib/uplink/ws_uplink.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index e75caa3335..8d8efc395a 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -76,7 +76,7 @@ namespace uplink { WS_uplink::WS_uplink(Config config) : config_{std::move(config)}, inet_{*config_.inet}, - id_{inet_.link_addr().to_string()}, + id_{__arch_system_info().uuid}, parser_({this, &WS_uplink::handle_transport}), heartbeat_timer({this, &WS_uplink::on_heartbeat_timer}) { From a1ee6433fe0f9fd198794c063e9c0d19ba0747a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 26 Jun 2018 14:08:47 +0200 Subject: [PATCH 461/723] net: Reset TCP stream on_close to avoid reentrance #1821 --- api/net/tcp/stream.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/api/net/tcp/stream.hpp b/api/net/tcp/stream.hpp index 873cd79a22..fca8095ee6 100644 --- a/api/net/tcp/stream.hpp +++ b/api/net/tcp/stream.hpp @@ -101,6 +101,7 @@ namespace net::tcp void close() override { auto onclose = std::move(this->m_on_close); + this->m_on_close.reset(); m_tcp->reset_callbacks(); m_tcp->close(); if (onclose) onclose(); From ad602a89e3f97193f777f362576a9f46512cacf5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 26 Jun 2018 16:09:12 +0200 Subject: [PATCH 462/723] util: MemoryRingBuffer now accepts void* --- api/util/ringbuffer.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/util/ringbuffer.hpp b/api/util/ringbuffer.hpp index 781d121e8a..27d1a47524 100644 --- a/api/util/ringbuffer.hpp +++ b/api/util/ringbuffer.hpp @@ -152,8 +152,8 @@ class RingBuffer { } protected: - RingBuffer(char* rbuffer, int size) - : cap(size), buffer(rbuffer) + RingBuffer(void* rbuffer, int size) + : cap(size), buffer((char*) rbuffer) { assert(size > 0); } @@ -183,10 +183,10 @@ class HeapRingBuffer : public RingBuffer { class MemoryRingBuffer : public RingBuffer { public: - MemoryRingBuffer(char* location, int size) + MemoryRingBuffer(void* location, int size) : RingBuffer(location, size) {} - MemoryRingBuffer(char* loc, const int cap, + MemoryRingBuffer(void* loc, const int cap, int start, int end, int used) : RingBuffer(loc, cap) { From 45971651e151e7e09d050804feb2401a808a108f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 26 Jun 2018 16:10:03 +0200 Subject: [PATCH 463/723] kernel: Fix SystemLog overestimating its size --- api/kernel/memory.hpp | 3 -- src/plugins/system_log.cpp | 36 +++++++++---------- .../integration/LiveUpdate/CMakeLists.txt | 4 +-- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/api/kernel/memory.hpp b/api/kernel/memory.hpp index 4cde19f951..dc08190963 100644 --- a/api/kernel/memory.hpp +++ b/api/kernel/memory.hpp @@ -310,12 +310,9 @@ namespace mem { using namespace util::bitops; const auto flags = os::mem::Access::read | os::mem::Access::write; // setup @dst as new virt area for @src - printf("mem::map\n"); os::mem::map({dst, src, flags, size}, label); // unpresent @src - printf("mem::protect\n"); os::mem::protect(src, size, os::mem::Access::none); - printf("done\n"); } }} diff --git a/src/plugins/system_log.cpp b/src/plugins/system_log.cpp index 6220bb7132..9619ec307b 100644 --- a/src/plugins/system_log.cpp +++ b/src/plugins/system_log.cpp @@ -10,15 +10,15 @@ struct Log_buffer { uint32_t flags; char vla[0]; - static const int PADDING = 32; static const uint64_t MAGIC = 0xDEADC0DEDEADC0DE; MemoryRingBuffer* get_mrb() { return reinterpret_cast(&vla[0]); } }; -static FixedRingBuffer<32768> temp_mrb; -#define MRB_LOG_SIZE (1 << 20) +static FixedRingBuffer<16384> temp_mrb; +#define MRB_AREA_SIZE (65536) // 64kb +#define MRB_LOG_SIZE (MRB_AREA_SIZE - sizeof(MemoryRingBuffer) - sizeof(Log_buffer)) static MemoryRingBuffer* mrb = nullptr; static inline RingBuffer* get_mrb() { @@ -26,23 +26,21 @@ static inline RingBuffer* get_mrb() return &temp_mrb; } -inline char* get_ringbuffer_loc() +inline static char* get_system_log_loc() { #ifdef ARCH_x86_64 - return (char*) ((1ull << 46) - MRB_LOG_SIZE); + return (char*) ((1ull << 46) - MRB_AREA_SIZE); #else - return (char*) OS::liveupdate_storage_area() - MRB_LOG_SIZE; + return (char*) OS::liveupdate_storage_area() - MRB_AREA_SIZE; #endif } - -inline char* get_system_log_loc() +inline static auto* get_ringbuffer_data() { - return get_ringbuffer_loc() - sizeof(Log_buffer) - Log_buffer::PADDING; + return get_system_log_loc() + sizeof(Log_buffer) + sizeof(MemoryRingBuffer); } - -inline Log_buffer& get_log_buffer() +inline static auto& get_log_buffer() { - return *((Log_buffer*) get_system_log_loc()); + return *(Log_buffer*) get_system_log_loc(); } uint32_t SystemLog::get_flags() @@ -81,11 +79,11 @@ void SystemLog::initialize() #ifdef ARCH_x86_64 using namespace util::bitops; - const size_t size = MRB_LOG_SIZE - 4096; - const uintptr_t syslog_area = (uintptr_t) get_ringbuffer_loc() - 4096; + const uintptr_t syslog_area = (uintptr_t) get_system_log_loc(); const uintptr_t lu_phys = os::mem::virt_to_phys((uintptr_t) OS::liveupdate_storage_area()); // move systemlog to high memory and unpresent physical - os::mem::virtual_move(lu_phys - size, size, syslog_area, "SystemLog"); + os::mem::virtual_move(lu_phys - MRB_AREA_SIZE, MRB_AREA_SIZE, + syslog_area, "SystemLog"); #endif auto& buffer = get_log_buffer(); @@ -94,19 +92,21 @@ void SystemLog::initialize() // There isn't one, so we have to create if(buffer.magic != Log_buffer::MAGIC) { - new (mrb) MemoryRingBuffer(get_ringbuffer_loc(), MRB_LOG_SIZE); + new (mrb) MemoryRingBuffer(get_ringbuffer_data(), MRB_LOG_SIZE); buffer.magic = Log_buffer::MAGIC; buffer.capacity = mrb->capacity(); buffer.flags = 0; - INFO2("Created @ %p (%i kB)", mrb->data(), mrb->capacity() / 1024); + INFO2("Created @ %p (%i kB)", get_system_log_loc(), mrb->capacity() / 1024); + INFO2("Data @ %p (%i bytes)", mrb->data(), mrb->capacity()); } // Correct magic means (hopefully) existing system log else { auto* state = (int*)(&buffer.vla); + assert(state[0] >= 16); - new (mrb) MemoryRingBuffer(get_ringbuffer_loc(), + new (mrb) MemoryRingBuffer(get_ringbuffer_data(), state[0], state[1], state[2], state[3]); INFO2("Restored @ %p (%i kB) Flags: 0x%x", diff --git a/test/kernel/integration/LiveUpdate/CMakeLists.txt b/test/kernel/integration/LiveUpdate/CMakeLists.txt index 82c965e700..251d546641 100644 --- a/test/kernel/integration/LiveUpdate/CMakeLists.txt +++ b/test/kernel/integration/LiveUpdate/CMakeLists.txt @@ -28,7 +28,7 @@ set(DRIVERS ) set(PLUGINS - #system_log + system_log ) # STATIC LIBRARIES: @@ -37,7 +37,7 @@ set(LIBRARIES ) # disable serial output -#set(default_stdout OFF) +set(default_stdout OFF) # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) From e20d10261b08f1aa79559a3ba977dcfa7addfd47 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 26 Jun 2018 16:53:17 +0200 Subject: [PATCH 464/723] microlb: Avoid losing connection when assigning to a node fails --- lib/microLB/micro_lb/balancer.cpp | 15 +++++++++++---- lib/microLB/micro_lb/balancer.hpp | 3 ++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/microLB/micro_lb/balancer.cpp b/lib/microLB/micro_lb/balancer.cpp index 5236cdf9d5..2d66368cda 100644 --- a/lib/microLB/micro_lb/balancer.cpp +++ b/lib/microLB/micro_lb/balancer.cpp @@ -73,9 +73,16 @@ namespace microLB auto& client = queue.front(); if (client.conn->is_connected()) { // NOTE: explicitly want to copy buffers - if (nodes.assign(std::move(client.conn), client.readq)) { + net::Stream_ptr rval = + nodes.assign(std::move(client.conn), client.readq); + if (rval == nullptr) { + // done with this queue item queue.pop_front(); } + else { + // put connection back in queue item + client.conn = std::move(rval); + } } else { queue.pop_front(); @@ -153,7 +160,7 @@ namespace microLB } } } - bool Nodes::assign(net::Stream_ptr conn, queue_vector_t& readq) + net::Stream_ptr Nodes::assign(net::Stream_ptr conn, queue_vector_t& readq) { for (size_t i = 0; i < nodes.size(); i++) { @@ -173,10 +180,10 @@ namespace microLB LBOUT("*** Flushing %lu bytes\n", buffer->size()); session.outgoing->write(buffer); } - return true; + return nullptr; } } - return false; + return std::move(conn); } size_t Nodes::size() const noexcept { return nodes.size(); diff --git a/lib/microLB/micro_lb/balancer.hpp b/lib/microLB/micro_lb/balancer.hpp index 4091cbfa67..0a83abc9cb 100644 --- a/lib/microLB/micro_lb/balancer.hpp +++ b/lib/microLB/micro_lb/balancer.hpp @@ -77,7 +77,8 @@ namespace microLB template void add_node(Args&&... args); void create_connections(int total); - bool assign(net::Stream_ptr, queue_vector_t&); + // returns the connection back if the operation fails + net::Stream_ptr assign(net::Stream_ptr, queue_vector_t&); Session& create_session(bool talk, net::Stream_ptr inc, net::Stream_ptr out); void close_session(int, bool timeout = false); Session& get_session(int); From 5e52722fdff1ced16392f21f948f25661d1516ca Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 26 Jun 2018 17:26:05 +0200 Subject: [PATCH 465/723] test: Try to wait for MSL end in microLB test --- test/net/integration/microLB/service.cpp | 9 +++++++++ test/net/integration/microLB/test.py | 11 ++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/test/net/integration/microLB/service.cpp b/test/net/integration/microLB/service.cpp index d76439a1e6..0cc210aad2 100644 --- a/test/net/integration/microLB/service.cpp +++ b/test/net/integration/microLB/service.cpp @@ -17,9 +17,18 @@ #include #include +#include +#include void Service::start() { static auto* balancer = microLB::Balancer::from_config(); printf("MicroLB ready for test\n"); + auto& inet = net::Super_stack::get(0); + inet.tcp().set_MSL(std::chrono::seconds(2)); + + Timers::oneshot(std::chrono::seconds(5), + [] (int) { + printf("TCP MSL ended (4 seconds)\n"); + }); } diff --git a/test/net/integration/microLB/test.py b/test/net/integration/microLB/test.py index a899eae64e..9089f98bc2 100755 --- a/test/net/integration/microLB/test.py +++ b/test/net/integration/microLB/test.py @@ -4,6 +4,7 @@ import sys import subprocess import thread +import time import atexit includeos_src = os.environ.get('INCLUDEOS_SRC', @@ -21,6 +22,7 @@ def validateRequest(expected = ""): # start nodeJS pro = subprocess.Popen(["nodejs", "server.js"], stdout=subprocess.PIPE) +requests_completed = False def startBenchmark(line): print " starting test " assert validateRequest("6001") @@ -32,7 +34,13 @@ def startBenchmark(line): assert validateRequest("6002") assert validateRequest("6003") assert validateRequest("6004") - vm._on_success("SUCCESS") + print "Waiting for TCP MSL end..." + global requests_completed + requests_completed = True + return True +def mslEnded(line): + if requests_completed: + vm._on_success("SUCCESS") return True @atexit.register @@ -47,6 +55,7 @@ def cleanup(): # Add custom event for testing server vm.on_output("MicroLB ready for test", startBenchmark) +vm.on_output("TCP MSL ended", mslEnded) # Boot the VM, taking a timeout as parameter vm.cmake().boot(20).clean() From e2d091752f1ac0a1869e9215e313ad39c54d4ebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 27 Jun 2018 12:20:12 +0200 Subject: [PATCH 466/723] util: set inplace delg dtor ptr to null on move to make it appear empty #1821 --- api/net/tcp/stream.hpp | 1 - api/util/delegate.hpp | 4 ++-- test/util/unit/delegate.cpp | 7 +++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/api/net/tcp/stream.hpp b/api/net/tcp/stream.hpp index fca8095ee6..873cd79a22 100644 --- a/api/net/tcp/stream.hpp +++ b/api/net/tcp/stream.hpp @@ -101,7 +101,6 @@ namespace net::tcp void close() override { auto onclose = std::move(this->m_on_close); - this->m_on_close.reset(); m_tcp->reset_callbacks(); m_tcp->close(); if (onclose) onclose(); diff --git a/api/util/delegate.hpp b/api/util/delegate.hpp index 3087322b4e..4dc1795f86 100644 --- a/api/util/delegate.hpp +++ b/api/util/delegate.hpp @@ -279,7 +279,7 @@ template< copy_ptr_{ other.copy_ptr_ }, destructor_ptr_{ other.destructor_ptr_ } { - other.destructor_ptr_ = [](storage_t&) -> void {}; + other.destructor_ptr_ = nullptr; } inplace& operator= (const inplace& other) @@ -311,7 +311,7 @@ template< copy_ptr_ = other.copy_ptr_; destructor_ptr_ = other.destructor_ptr_; - other.destructor_ptr_ = [](storage_t&) -> void {}; + other.destructor_ptr_ = nullptr; } return *this; } diff --git a/test/util/unit/delegate.cpp b/test/util/unit/delegate.cpp index dc3a8998fa..87dda664e2 100644 --- a/test/util/unit/delegate.cpp +++ b/test/util/unit/delegate.cpp @@ -223,6 +223,13 @@ CASE("A delegate can be moved") size_t ret = del_b(); EXPECT(ret == v.size()); + EXPECT_NOT(del_a); + + user_class usr(1); + auto usr_del = usr.get_del(); + EXPECT(usr_del); + auto usr_del2 = std::move(usr_del); + EXPECT_NOT(usr_del); } CASE("A delegate can be constructed with a class member function pointer") From b353683e4a3586d425760ece5c0d9feaef631b73 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 00:11:53 +0200 Subject: [PATCH 467/723] microLB: Replace auto with explicit params in lambda --- lib/microLB/micro_lb/openssl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/microLB/micro_lb/openssl.cpp b/lib/microLB/micro_lb/openssl.cpp index 13deb23e2a..0c638a8be5 100644 --- a/lib/microLB/micro_lb/openssl.cpp +++ b/lib/microLB/micro_lb/openssl.cpp @@ -15,7 +15,7 @@ namespace microLB : nodes(), netin(in), netout(out), signal({this, &Balancer::handle_queue}) { fs::memdisk().init_fs( - [] (auto err, auto&) { + [] (fs::error_t err, fs::File_system&) { assert(!err); }); From c71835a973aba8eeae4291c4de4f78de54af8087 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 00:17:27 +0200 Subject: [PATCH 468/723] pci: Refactor port IO inline asm --- api/hw/pci.hpp | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/api/hw/pci.hpp b/api/hw/pci.hpp index 57fac34f0e..d19d8d1241 100644 --- a/api/hw/pci.hpp +++ b/api/hw/pci.hpp @@ -24,72 +24,59 @@ namespace hw { - typedef uint16_t port_t; - - static inline uint8_t inp(port_t port) + static inline uint8_t inp(uint16_t port) { uint8_t ret; #if defined(ARCH_x86) - __asm__ volatile("xorl %eax,%eax"); - __asm__ volatile("inb %%dx,%%al" - :"=a"(ret) - :"d"(port)); + asm volatile("inb %w1,%b0" : "=a"(ret) : "Nd"(port)); #else #error "inp() not implemented for selected arch" #endif return ret; } - static inline uint16_t inpw(port_t port) + static inline uint16_t inpw(uint16_t port) { uint16_t ret; #if defined(ARCH_x86) - __asm__ volatile("xorl %eax,%eax"); - __asm__ volatile("inw %%dx,%%ax" - :"=a"(ret) - :"d"(port)); + asm volatile("inw %w1,%w0" : "=a"(ret) : "Nd"(port)); #else #error "inpw() not implemented for selected arch" #endif return ret; } - static inline uint32_t inpd(port_t port) + static inline uint32_t inpd(uint16_t port) { uint32_t ret; #if defined(ARCH_x86) - __asm__ volatile("xorl %eax,%eax"); - __asm__ volatile("inl %%dx,%%eax" - :"=a"(ret) - :"d"(port)); + asm volatile("inl %w1,%d0" : "=a"(ret) : "Nd"(port)); #else #error "inpd() not implemented for selected arch" #endif - return ret; } - - static inline void outp(port_t port, uint8_t data) + static inline void outp(uint16_t port, uint8_t data) { #if defined(ARCH_x86) - __asm__ volatile ("outb %%al,%%dx"::"a" (data), "d"(port)); + asm volatile ("outb %b0,%w1" :: "a"(data), "Nd"(port)); #else #error "outp() not implemented for selected arch" #endif } - static inline void outpw(port_t port, uint16_t data) + static inline void outpw(uint16_t port, uint16_t data) { #if defined(ARCH_x86) - __asm__ volatile ("outw %%ax,%%dx"::"a" (data), "d"(port)); + asm volatile ("outw %w0,%w1" :: "a" (data), "Nd"(port)); #else #error "outpw() not implemented for selected arch" #endif } - static inline void outpd(port_t port, uint32_t data) + static inline void outpd(uint16_t port, uint32_t data) { #if defined(ARCH_x86) - __asm__ volatile ("outl %%eax,%%dx"::"a" (data), "d"(port)); + asm volatile ("outl %d0,%w1" :: "a" (data), "Nd"(port)); #else #error "outpd() not implemented for selected arch" #endif From ea5dfcd89e15d0ae043f5225053bed12dec7b5b8 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 00:18:52 +0200 Subject: [PATCH 469/723] pci: Work to run better with g++ --- api/hw/pci_device.hpp | 4 ++-- src/drivers/e1000.cpp | 2 +- src/hw/msi.cpp | 2 +- src/hw/pci_device.cpp | 45 +++++++++++++++++++++---------------------- src/virtio/virtio.cpp | 2 +- 5 files changed, 27 insertions(+), 28 deletions(-) diff --git a/api/hw/pci_device.hpp b/api/hw/pci_device.hpp index 18a427745c..d8e920482f 100644 --- a/api/hw/pci_device.hpp +++ b/api/hw/pci_device.hpp @@ -149,7 +149,7 @@ struct msix_t; explicit PCI_Device(const uint16_t pci_addr, const uint32_t, const uint32_t); //! @brief Read from device with implicit pci_address (e.g. used by Nic) - uint32_t read_dword(const uint8_t reg) noexcept; + uint32_t read32(const uint8_t reg) noexcept; //! @brief Read from device with explicit pci_addr static uint32_t read_dword(const uint16_t pci_addr, const uint8_t reg) noexcept; @@ -287,7 +287,7 @@ struct msix_t; int m_iobase = -1; std::array m_resources; - pcicap_t caps[PCI_CAP_ID_MAX+1]; + std::array caps; // has msix support if not null msix_t* msix = nullptr; diff --git a/src/drivers/e1000.cpp b/src/drivers/e1000.cpp index 81b56feb80..deef84b0c0 100644 --- a/src/drivers/e1000.cpp +++ b/src/drivers/e1000.cpp @@ -99,7 +99,7 @@ e1000::e1000(hw::PCI_Device& d, uint16_t mtu) : // 21 = lots of times (???) // 16 = USB // 18 = e1000 I217 - uint32_t value = d.read_dword(PCI::CONFIG_INTR); + uint32_t value = d.read32(PCI::CONFIG_INTR); uint8_t real_irq = value & 0xFF; assert(real_irq != 0xFF); diff --git a/src/hw/msi.cpp b/src/hw/msi.cpp index 8cdc8c2428..b7c09abb88 100644 --- a/src/hw/msi.cpp +++ b/src/hw/msi.cpp @@ -94,7 +94,7 @@ namespace hw * Software calculates the base address of the MSI-X PBA using the same process * with the PBA Offset / PBA BIR register. **/ - auto bar = dev.read_dword(offset); + auto bar = dev.read32(offset); auto capbar_off = bar & ~MSIX_BIR_MASK; bar &= MSIX_BIR_MASK; diff --git a/src/hw/pci_device.cpp b/src/hw/pci_device.cpp index 632ba6f70f..9a16471ccf 100644 --- a/src/hw/pci_device.cpp +++ b/src/hw/pci_device.cpp @@ -74,13 +74,13 @@ namespace hw { { //Read the current BAR register uint32_t reg = PCI::CONFIG_BASE_ADDR_0 + (bar << 2); - uint32_t value = read_dword(reg); + uint32_t value = read32(reg); if (!value) continue; //Write all 1's to the register, to get the length value (osdev) write_dword(reg, 0xFFFFFFFF); - uint32_t len = read_dword(reg); + uint32_t len = read32(reg); //Put the value back write_dword(reg, value); @@ -113,7 +113,7 @@ namespace hw { : pci_addr_{pci_addr}, device_id_{device_id} { // set master, mem and io flags - uint32_t cmd = read_dword(PCI_CMD_REG); + uint32_t cmd = read32(PCI_CMD_REG); cmd |= PCI_COMMAND_MASTER | PCI_COMMAND_MEM | PCI_COMMAND_IO; write_dword(PCI_CMD_REG, cmd); @@ -152,6 +152,16 @@ namespace hw { if (classcode() == PCI::classcode::BRIDGE) return; } + uint32_t PCI_Device::read_dword(const uint16_t pci_addr, const uint8_t reg) noexcept { + PCI::msg req; + + req.data = 0x80000000; + req.addr = pci_addr; + req.reg = reg; + + outpd(PCI::CONFIG_ADDR, 0x80000000 | req.data); + return inpd(PCI::CONFIG_DATA); + } void PCI_Device::write_dword(const uint8_t reg, const uint32_t value) noexcept { PCI::msg req; @@ -163,7 +173,7 @@ namespace hw { outpd(PCI::CONFIG_DATA, value); } - uint32_t PCI_Device::read_dword(const uint8_t reg) noexcept { + uint32_t PCI_Device::read32(const uint8_t reg) noexcept { PCI::msg req; req.data = 0x80000000; @@ -174,6 +184,7 @@ namespace hw { return inpd(PCI::CONFIG_DATA); } + __attribute__((noinline)) uint16_t PCI_Device::read16(const uint8_t reg) noexcept { PCI::msg req; req.data = 0x80000000; @@ -181,7 +192,8 @@ namespace hw { req.reg = reg; outpd(PCI::CONFIG_ADDR, 0x80000000 | req.data); - return inpw(PCI::CONFIG_DATA + (reg & 2)); + uint16_t data = inpw(PCI::CONFIG_DATA + (reg & 2)); + return data; } void PCI_Device::write16(const uint8_t reg, const uint16_t value) noexcept { PCI::msg req; @@ -193,36 +205,24 @@ namespace hw { outpw(PCI::CONFIG_DATA + (reg & 2), value); } - uint32_t PCI_Device::read_dword(const uint16_t pci_addr, const uint8_t reg) noexcept { - PCI::msg req; - - req.data = 0x80000000; - req.addr = pci_addr; - req.reg = reg; - - outpd(PCI::CONFIG_ADDR, 0x80000000 | req.data); - return inpd(PCI::CONFIG_DATA); - } - union capability_t { struct { uint8_t id; uint8_t next; - char data[2]; + uint16_t data; }; uint32_t capd; }; void PCI_Device::parse_capabilities() { - /// FROM http://wiki.osdev.org/PCI - memset(caps, 0, sizeof(caps)); - + caps = {}; // the capability list is only available if bit 4 // in the status register is set uint16_t status = read16(PCI_STATUS_REG); + //printf("read16 %#x status %#x\n", PCI_STATUS_REG, status); if ((status & 0x10) == 0) return; // this offset works for non-cardbus bridges uint32_t offset = 0x34; @@ -232,10 +232,9 @@ namespace hw { while (offset) { capability_t cap; - cap.capd = read_dword(offset); - assert (cap.id <= PCI_CAP_ID_MAX); + cap.capd = read32(offset); // remember capability - this->caps[cap.id] = offset; + this->caps.at(cap.id) = offset; // go to next cap offset = cap.next; } diff --git a/src/virtio/virtio.cpp b/src/virtio/virtio.cpp index fe278e6bd0..08b7fed1ad 100644 --- a/src/virtio/virtio.cpp +++ b/src/virtio/virtio.cpp @@ -159,7 +159,7 @@ void Virtio::reset() { uint8_t Virtio::get_legacy_irq() { // Get legacy IRQ from PCI - uint32_t value = _pcidev.read_dword(PCI::CONFIG_INTR); + uint32_t value = _pcidev.read32(PCI::CONFIG_INTR); if ((value & 0xFF) != 0xFF) { return value & 0xFF; } From a4833c67764559007b87688db9cb8b255c9b7023 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 10:24:05 +0200 Subject: [PATCH 470/723] pci: Work-around for inline asm in clang --- api/hw/pci.hpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/api/hw/pci.hpp b/api/hw/pci.hpp index d19d8d1241..c7bcc5ddb2 100644 --- a/api/hw/pci.hpp +++ b/api/hw/pci.hpp @@ -28,7 +28,7 @@ namespace hw { { uint8_t ret; #if defined(ARCH_x86) - asm volatile("inb %w1,%b0" : "=a"(ret) : "Nd"(port)); + asm volatile("inb %1,%0" : "=a"(ret) : "Nd"(port)); #else #error "inp() not implemented for selected arch" #endif @@ -39,7 +39,7 @@ namespace hw { { uint16_t ret; #if defined(ARCH_x86) - asm volatile("inw %w1,%w0" : "=a"(ret) : "Nd"(port)); + asm volatile("inw %1,%0" : "=a"(ret) : "Nd"(port)); #else #error "inpw() not implemented for selected arch" #endif @@ -50,7 +50,7 @@ namespace hw { { uint32_t ret; #if defined(ARCH_x86) - asm volatile("inl %w1,%d0" : "=a"(ret) : "Nd"(port)); + asm volatile("inl %1,%0" : "=a"(ret) : "Nd"(port)); #else #error "inpd() not implemented for selected arch" #endif @@ -60,7 +60,7 @@ namespace hw { static inline void outp(uint16_t port, uint8_t data) { #if defined(ARCH_x86) - asm volatile ("outb %b0,%w1" :: "a"(data), "Nd"(port)); + asm volatile ("outb %0,%1" :: "a"(data), "Nd"(port)); #else #error "outp() not implemented for selected arch" #endif @@ -68,7 +68,7 @@ namespace hw { static inline void outpw(uint16_t port, uint16_t data) { #if defined(ARCH_x86) - asm volatile ("outw %w0,%w1" :: "a" (data), "Nd"(port)); + asm volatile ("outw %0,%1" :: "a" (data), "Nd"(port)); #else #error "outpw() not implemented for selected arch" #endif @@ -76,7 +76,7 @@ namespace hw { static inline void outpd(uint16_t port, uint32_t data) { #if defined(ARCH_x86) - asm volatile ("outl %d0,%w1" :: "a" (data), "Nd"(port)); + asm volatile ("outl %0,%1" :: "a" (data), "Nd"(port)); #else #error "outpd() not implemented for selected arch" #endif From 18915d9039fc8c8d82ad3af4237bb65d20320b01 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 13:01:04 +0200 Subject: [PATCH 471/723] fstatat: Make sure there is room for null-terminator --- src/musl/fstatat.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/musl/fstatat.cpp b/src/musl/fstatat.cpp index bf9ec58bbe..24434bcde8 100644 --- a/src/musl/fstatat.cpp +++ b/src/musl/fstatat.cpp @@ -10,9 +10,9 @@ static long sys_fstatat(int fd, const char *path, struct stat *buf, int flag) { if (fd == AT_FDCWD) { - char cwd_buf[PATH_MAX]; + char cwd_buf[PATH_MAX-1]; char abs_path[PATH_MAX]; - if (sys_getcwd(cwd_buf, PATH_MAX) > 0) { + if (sys_getcwd(cwd_buf, PATH_MAX-1) > 0) { snprintf(abs_path, PATH_MAX, "%s/%s", cwd_buf, path); } return sys_stat(abs_path, buf); From 0b4e8565948600cac629e80399b24ac51dc4f282 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 13:01:42 +0200 Subject: [PATCH 472/723] kernel: Fix warning in panic() --- src/kernel/syscalls.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index b48fd736b9..44667b68e4 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -188,10 +188,10 @@ void panic_epilogue(const char* why) case OS::Panic_action::halt: while (1) OS::halt(); case OS::Panic_action::shutdown: - extern void __arch_poweroff(); + extern __attribute__((noreturn)) void __arch_poweroff(); __arch_poweroff(); + [[fallthrough]]; // needed for g++ bug case OS::Panic_action::reboot: - default: OS::reboot(); } From fd40ddf0f10cf5e7df695e754d1be8b9d27ea1a9 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 13:02:32 +0200 Subject: [PATCH 473/723] fd: Fix warning in unix_fd --- api/posix/unix_fd.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/posix/unix_fd.hpp b/api/posix/unix_fd.hpp index 969cf28a91..9e99e654a2 100644 --- a/api/posix/unix_fd.hpp +++ b/api/posix/unix_fd.hpp @@ -37,8 +37,8 @@ class Unix_FD : public SockFD { int close() override; private: Impl* impl = nullptr; - const int type_ [[maybe_unused]]; // it's probably gonna be necessary - // to tell if socket is stream or dgram + const int type_; // it's probably gonna be necessary + // to tell if socket is stream or dgram long set_impl_if_needed(const struct sockaddr* addr, socklen_t addrlen); }; From eb8fb7aedb92961c4b85090fdf55a60f048e2a24 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 13:02:56 +0200 Subject: [PATCH 474/723] x86: Fix warnings in paging.hpp --- api/arch/x86/paging.hpp | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/api/arch/x86/paging.hpp b/api/arch/x86/paging.hpp index 6b41660de0..f14480bcd8 100644 --- a/api/arch/x86/paging.hpp +++ b/api/arch/x86/paging.hpp @@ -468,8 +468,9 @@ class Page_table { **/ void map_all(uintptr_t phys, Pflag flags) { - if (! is_range_aligned(phys)) - PG_PRINT(" Physical addr 0x%lx is not range aligned \n", phys); + if (!is_range_aligned(phys)) { + printf(" Physical addr 0x%lx is not range aligned \n", phys); + } Expects(is_range_aligned(phys)); for(auto& page : this->tbl_) { @@ -506,8 +507,9 @@ class Page_table { { Expects(entry >= tbl_.begin() && entry < tbl_.end()); auto new_entry = addr_of(*entry) | (flags & allowed_flags); - if (!is_page_aligned(addr_of(new_entry))) + if (!is_page_aligned(addr_of(new_entry))) { printf("%p is not page aligned \n", (void*)addr_of(new_entry)); + } Expects(is_page_aligned(addr_of(new_entry))); // Can only make pages and page dirs present. E.g. no unmapped PML4 entry. @@ -622,8 +624,9 @@ class Page_table { req.size = page_size; req.page_sizes = page_size; - if (addr_of(*ent) != req.phys) - PG_PRINT("Couldn't set address: req. expected 0x%lx, got 0x%lx\n", req.phys, addr_of(*ent)); + if (addr_of(*ent) != req.phys) { + printf("Couldn't set address: req. expected 0x%lx, got 0x%lx\n", req.phys, addr_of(*ent)); + } Ensures(addr_of(*ent) == req.phys); return req; } From 6dbd56eecd589feff04a489e48d467973460ceba Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 13:10:08 +0200 Subject: [PATCH 475/723] solo5: Cleanup start phase --- api/kernel/os.hpp | 2 +- src/platform/x86_solo5/kernel_start.cpp | 79 +++++++++++++------------ src/platform/x86_solo5/os.cpp | 6 +- src/platform/x86_solo5/platform.cpp | 9 +-- 4 files changed, 49 insertions(+), 47 deletions(-) diff --git a/api/kernel/os.hpp b/api/kernel/os.hpp index cf151b2556..b6d8a5f259 100644 --- a/api/kernel/os.hpp +++ b/api/kernel/os.hpp @@ -243,7 +243,7 @@ class OS { /** Initialize platform, devices etc. */ static void start(uint32_t boot_magic, uint32_t boot_addr); - static void start(char *cmdline, uintptr_t mem_size); + static void start(const char* cmdline); /** Initialize common subsystems, call Service::start */ static void post_start(); diff --git a/src/platform/x86_solo5/kernel_start.cpp b/src/platform/x86_solo5/kernel_start.cpp index 3c1b87c22d..9f2b72a87b 100644 --- a/src/platform/x86_solo5/kernel_start.cpp +++ b/src/platform/x86_solo5/kernel_start.cpp @@ -10,10 +10,6 @@ extern "C" { } extern void __platform_init(); -void __arch_subscribe_irq(unsigned char) {} // for now - -char cmdline[256]; -uintptr_t mem_size; extern "C" { void __init_sanity_checks(); @@ -26,52 +22,57 @@ extern "C" { uintptr_t _end; void set_stack(); void* get_cpu_ebp(); +} - void kernel_start() - { +static char temp_cmdline[1024]; +static uintptr_t mem_size = 0; - // generate checksums of read-only areas etc. - __init_sanity_checks(); +extern "C" +void kernel_start() +{ + // generate checksums of read-only areas etc. + __init_sanity_checks(); - // Determine where free memory starts - uintptr_t free_mem_begin = reinterpret_cast(&_end); + // Determine where free memory starts + uintptr_t free_mem_begin = reinterpret_cast(&_end); - // Preserve symbols from the ELF binary - free_mem_begin += _move_symbols(free_mem_begin); + // Preserve symbols from the ELF binary + free_mem_begin += _move_symbols(free_mem_begin); - // Do not zero out all solo5 global variables!! == don't touch the BSS - //_init_bss(); + // Do not zero out all solo5 global variables!! == don't touch the BSS + //_init_bss(); - // Initialize heap - // XXX: this is dangerous as solo5 might be doing malloc()'s using it's own - // idea of a heap. Luckily there is no malloc instance at solo5/kernel/[ukvm|virtio|muen], - // so might be OK (for now). - OS::init_heap(free_mem_begin, mem_size); + // Initialize heap + // XXX: this is dangerous as solo5 might be doing malloc()'s using it's own + // idea of a heap. Luckily there is no malloc instance at solo5/kernel/[ukvm|virtio|muen], + // so might be OK (for now). + OS::init_heap(free_mem_begin, mem_size); - _init_elf_parser(); + _init_elf_parser(); - // Initialize system calls - _init_syscalls(); + // Initialize system calls + _init_syscalls(); - // Initialize OS including devices - OS::start(cmdline, mem_size); - OS::post_start(); + // Initialize OS including devices + OS::start(temp_cmdline); + OS::post_start(); - // Starting event loop from here allows us to profile OS::start - OS::event_loop(); - } + // Starting event loop from here allows us to profile OS::start + OS::event_loop(); +} - int solo5_app_main(char *_cmdline) - { - // cmdline is stored at 0x6000 by ukvm which is used by includeos. Move it fast. - strncpy(cmdline, _cmdline, 256); +extern "C" +int solo5_app_main(char* cmdline) +{ + // cmdline is stored at 0x6000 by ukvm which is used by includeos. Move it fast. + strncpy(temp_cmdline, cmdline, sizeof(temp_cmdline)-1); + temp_cmdline[sizeof(temp_cmdline)-1] = 0; - // solo5 sets the stack to be at the end of memory, so let's use that as - // our memory size (before we change). - mem_size = (uintptr_t)get_cpu_ebp(); + // solo5 sets the stack to be at the end of memory, so let's use that as + // our memory size (before we change). + mem_size = (uintptr_t) get_cpu_ebp(); - // set the stack location to its new includeos location, and call kernel_start - set_stack(); - return 0; - } + // set the stack location to its new includeos location, and call kernel_start + set_stack(); + return 0; } diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index d0d125cc2e..dabe9d2c06 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -71,8 +71,10 @@ void OS::default_stdout(const char* str, const size_t len) solo5_console_write(str, len); } -void OS::start(char* _cmdline, uintptr_t mem_size) +void OS::start(const char* cmdline) { + OS::cmdline = cmdline; + // Initialize stdout handlers if(os_default_stdout) { OS::add_stdout(&OS::default_stdout); @@ -89,8 +91,6 @@ void OS::start(char* _cmdline, uintptr_t mem_size) /// initialize on page 7, 2 pages in size Statman::get().init(0x6000, 0x3000); - OS::cmdline = _cmdline; - // Call global ctors PROFILE("Global kernel constructors"); __run_ctors(&__init_array_start, &__init_array_end); diff --git a/src/platform/x86_solo5/platform.cpp b/src/platform/x86_solo5/platform.cpp index 4d80081cc8..03fe2355b3 100644 --- a/src/platform/x86_solo5/platform.cpp +++ b/src/platform/x86_solo5/platform.cpp @@ -6,13 +6,14 @@ void __arch_poweroff() while (true) asm ("cli; hlt"); } -void __platform_init(){ +void __platform_init() { // minimal CPU exception handlers already set by solo5/ukvm } -void __arch_reboot(){} -void __arch_enable_legacy_irq(unsigned char){} -void __arch_disable_legacy_irq(unsigned char){} +void __arch_reboot() {} +void __arch_enable_legacy_irq(unsigned char) {} +void __arch_disable_legacy_irq(unsigned char) {} +void __arch_subscribe_irq(unsigned char) {} void SMP::global_lock() noexcept {} void SMP::global_unlock() noexcept {} From e94ed660f01fb2ab95e1720625b44c65603e6cea Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 13:49:29 +0200 Subject: [PATCH 476/723] hw: Make ioport header compatible with g++ --- api/hw/ioport.hpp | 73 +++++++++++++------------------------- examples/starlight/vm.json | 2 +- 2 files changed, 26 insertions(+), 49 deletions(-) diff --git a/api/hw/ioport.hpp b/api/hw/ioport.hpp index 887cb0d520..702de2c977 100644 --- a/api/hw/ioport.hpp +++ b/api/hw/ioport.hpp @@ -23,84 +23,61 @@ namespace hw { - /** Receive a byte from port. - @param port : The port number to receive from - */ - static inline uint8_t inb(int port) + static inline uint8_t inb(uint16_t port) { - int ret; + uint8_t ret; #if defined(ARCH_x86) - asm volatile ("xorl %eax,%eax"); - asm volatile ("inb %%dx,%%al":"=a" (ret):"d"(port)); + asm volatile("inb %1,%0" : "=a"(ret) : "Nd"(port)); #else -#error "inb() not implemented for selected arch" +#error "inp() not implemented for selected arch" #endif return ret; } - /** Send a byte to port. - @param port : The port to send to - @param data : One byte of data to send to @param port - */ - static inline void outb(int port, uint8_t data) { + static inline uint16_t inw(uint16_t port) + { + uint16_t ret; #if defined(ARCH_x86) - asm volatile ("outb %%al,%%dx"::"a" (data), "d"(port)); + asm volatile("inw %1,%0" : "=a"(ret) : "Nd"(port)); #else -#error "outb() not implemented for selected arch" +#error "inpw() not implemented for selected arch" #endif + return ret; } - /** Receive a word from port. - @param port : The port number to receive from - */ - static inline uint16_t inw(int port) + static inline uint32_t inl(uint16_t port) { - int ret; + uint32_t ret; #if defined(ARCH_x86) - asm volatile ("xorl %eax,%eax"); - asm volatile ("inw %%dx,%%ax":"=a" (ret):"d"(port)); + asm volatile("inl %1,%0" : "=a"(ret) : "Nd"(port)); #else -#error "inw() not implemented for selected arch" +#error "inpd() not implemented for selected arch" #endif return ret; } - /** Send a word to port. - @param port : The port to send to - @param data : One word of data to send to @param port - */ - static inline void outw(int port, uint16_t data) { + static inline void outb(uint16_t port, uint8_t data) + { #if defined(ARCH_x86) - asm volatile ("outw %%ax,%%dx"::"a" (data), "d"(port)); + asm volatile ("outb %0,%1" :: "a"(data), "Nd"(port)); #else -#error "outw() not implemented for selected arch" +#error "outp() not implemented for selected arch" #endif } - - /** Receive a double-word from port. - @param port : The port number to receive from - */ - static inline uint32_t inl(int port) + static inline void outw(uint16_t port, uint16_t data) { - uint32_t ret; #if defined(ARCH_x86) - //asm volatile ("xorl %eax,%eax"); - asm volatile ("inl %%dx,%%eax":"=a" (ret):"d"(port)); + asm volatile ("outw %0,%1" :: "a" (data), "Nd"(port)); #else -#error "inw() not implemented for selected arch" +#error "outpw() not implemented for selected arch" #endif - return ret; } - - /** Send a double-word to port. - @param port : The port to send to - @param data : Double-word of data - */ - static inline void outl(int port, uint32_t data) { + static inline void outl(uint16_t port, uint32_t data) + { #if defined(ARCH_x86) - asm volatile ("outl %%eax,%%dx"::"a" (data), "d"(port)); + asm volatile ("outl %0,%1" :: "a" (data), "Nd"(port)); #else -#error "outw() not implemented for selected arch" +#error "outpd() not implemented for selected arch" #endif } diff --git a/examples/starlight/vm.json b/examples/starlight/vm.json index bf8213ec1c..a931e0561e 100644 --- a/examples/starlight/vm.json +++ b/examples/starlight/vm.json @@ -1,4 +1,4 @@ { - "mem" : 6, + "mem" : 12, "vga" : "qxl" } From 3d4c950e9d69e0c8016c6466a3f58a1fe7bdffbf Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 28 Jun 2018 13:49:54 +0200 Subject: [PATCH 477/723] fd: Fix clang-specific warning in unix_fd --- src/posix/unix_fd.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/posix/unix_fd.cpp b/src/posix/unix_fd.cpp index c93716a4ea..1a50c4fd88 100644 --- a/src/posix/unix_fd.cpp +++ b/src/posix/unix_fd.cpp @@ -20,6 +20,7 @@ long Unix_FD::set_impl_if_needed(const struct sockaddr* addr, socklen_t addrlen) { + (void) type_; if((addr == nullptr and addrlen == 0) and impl != nullptr) return 0; From c5415e3b91e4ec6e54225af460a08e2fb4214aa5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 29 Jun 2018 11:11:28 +0200 Subject: [PATCH 478/723] microLB: Remove redundant move --- lib/microLB/micro_lb/balancer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/microLB/micro_lb/balancer.cpp b/lib/microLB/micro_lb/balancer.cpp index 2d66368cda..69cdb7e0ca 100644 --- a/lib/microLB/micro_lb/balancer.cpp +++ b/lib/microLB/micro_lb/balancer.cpp @@ -183,7 +183,7 @@ namespace microLB return nullptr; } } - return std::move(conn); + return conn; } size_t Nodes::size() const noexcept { return nodes.size(); From 587a4f44de6c1caae28827a5b01e914f72298d92 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 29 Jun 2018 15:40:11 +0200 Subject: [PATCH 479/723] x86: Support old KVM clocks, re-enable --- api/kernel/cpuid.hpp | 18 +++++++++--------- src/kernel/cpuid.cpp | 4 ++-- src/platform/kvm/kvmclock.cpp | 26 ++++++++++++++++++++++---- src/platform/x86_pc/clocks.cpp | 27 +++++++++++++++------------ 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/api/kernel/cpuid.hpp b/api/kernel/cpuid.hpp index 3c56f8d708..96dafcb440 100644 --- a/api/kernel/cpuid.hpp +++ b/api/kernel/cpuid.hpp @@ -125,19 +125,19 @@ namespace CPUID bool is_intel_cpu() noexcept; bool has_feature(Feature f); - bool kvm_feature(unsigned id) noexcept; + bool kvm_feature(unsigned mask) noexcept; } //< CPUID -#define KVM_FEATURE_CLOCKSOURCE 0 -#define KVM_FEATURE_NOP_IO_DELAY 1 -#define KVM_FEATURE_MMU_OP 2 /* deprecated */ -#define KVM_FEATURE_CLOCKSOURCE2 3 -#define KVM_FEATURE_ASYNC_PF 4 -#define KVM_FEATURE_STEAL_TIME 5 -#define KVM_FEATURE_PV_EOI 6 -#define KVM_FEATURE_PV_UNHALT 7 +#define KVM_FEATURE_CLOCKSOURCE 0x1 +#define KVM_FEATURE_NOP_IO_DELAY 0x2 +#define KVM_FEATURE_MMU_OP 0x4 /* deprecated */ +#define KVM_FEATURE_CLOCKSOURCE2 0x8 +#define KVM_FEATURE_ASYNC_PF 0x10 +#define KVM_FEATURE_STEAL_TIME 0x20 +#define KVM_FEATURE_PV_EOI 0x40 +#define KVM_FEATURE_PV_UNHALT 0x80 #define KVM_FEATURE_CLOCKSOURCE_STABLE_BIT 24 #endif //< KERNEL_CPUID_HPP diff --git a/src/kernel/cpuid.cpp b/src/kernel/cpuid.cpp index 9e843a8234..1772dd2dfa 100644 --- a/src/kernel/cpuid.cpp +++ b/src/kernel/cpuid.cpp @@ -320,12 +320,12 @@ static unsigned kvm_function() noexcept return res.EAX; return 0; } -bool CPUID::kvm_feature(unsigned id) noexcept +bool CPUID::kvm_feature(unsigned mask) noexcept { unsigned func = kvm_function(); if (func == 0) return false; auto res = cpuid(func, 0); - return (res.EAX & (1 << id)) != 0; + return (res.EAX & mask) != 0; } CPUID::Feature_list CPUID::detect_features() { diff --git a/src/platform/kvm/kvmclock.cpp b/src/platform/kvm/kvmclock.cpp index 61057dc573..a91fe395a1 100644 --- a/src/platform/kvm/kvmclock.cpp +++ b/src/platform/kvm/kvmclock.cpp @@ -1,13 +1,12 @@ #include "kvmclock.hpp" #include #include +#include #include #include #include #include -#define MSR_KVM_WALL_CLOCK_NEW 0x4b564d00 -#define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01 using namespace x86; using namespace util::literals; @@ -32,10 +31,29 @@ static pvclock_wall_clock kvm_wall_clock; void KVM_clock::init() { + #define MSR_KVM_WALL_CLOCK_NEW 0x4b564d00 + #define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01 + uint32_t msr_kvm_wall_clock; + uint32_t msr_kvm_system_time; + + if (CPUID::kvm_feature(KVM_FEATURE_CLOCKSOURCE2)) + { + msr_kvm_wall_clock = MSR_KVM_WALL_CLOCK_NEW; + msr_kvm_system_time = MSR_KVM_SYSTEM_TIME_NEW; + } + else if (CPUID::kvm_feature(KVM_FEATURE_CLOCKSOURCE)) + { + msr_kvm_wall_clock = 0x11; + msr_kvm_system_time = 0x12; + } + else { + assert(0 && "No KVM clock support detected"); + } + auto wall_addr = (uintptr_t) &kvm_wall_clock; - CPU::write_msr(MSR_KVM_WALL_CLOCK_NEW, wall_addr); + CPU::write_msr(msr_kvm_wall_clock, wall_addr); auto vcpu_addr = (uintptr_t) &PER_CPU(vcpu_time); - CPU::write_msr(MSR_KVM_SYSTEM_TIME_NEW, vcpu_addr | 1); + CPU::write_msr(msr_kvm_system_time, vcpu_addr | 1); } KHz KVM_clock::get_tsc_khz() diff --git a/src/platform/x86_pc/clocks.cpp b/src/platform/x86_pc/clocks.cpp index a11e8b4e46..7955ca9d00 100644 --- a/src/platform/x86_pc/clocks.cpp +++ b/src/platform/x86_pc/clocks.cpp @@ -36,28 +36,31 @@ struct sysclock_t wall_time_t wall_time = nullptr; tsc_khz_t tsc_khz = nullptr; }; -static SMP::Array vcpu_clock; +static sysclock_t current_clock; namespace x86 { void Clocks::init() { - if (false && CPUID::kvm_feature(KVM_FEATURE_CLOCKSOURCE2)) + if (CPUID::kvm_feature(KVM_FEATURE_CLOCKSOURCE + | KVM_FEATURE_CLOCKSOURCE2)) { KVM_clock::init(); - PER_CPU(vcpu_clock).system_time = {&KVM_clock::system_time}; - PER_CPU(vcpu_clock).wall_time = {&KVM_clock::wall_clock}; - PER_CPU(vcpu_clock).tsc_khz = {&KVM_clock::get_tsc_khz}; - if (SMP::cpu_id() == 0) INFO("x86", "KVM PV clocks initialized"); + if (SMP::cpu_id() == 0) { + current_clock.system_time = {&KVM_clock::system_time}; + current_clock.wall_time = {&KVM_clock::wall_clock}; + current_clock.tsc_khz = {&KVM_clock::get_tsc_khz}; + INFO("x86", "KVM PV clocks initialized"); + } } else { // fallback with CMOS - PER_CPU(vcpu_clock).system_time = {&CMOS_clock::system_time}; - PER_CPU(vcpu_clock).wall_time = {&CMOS_clock::wall_clock}; - PER_CPU(vcpu_clock).tsc_khz = {&CMOS_clock::get_tsc_khz}; if (SMP::cpu_id() == 0) { + current_clock.system_time = {&CMOS_clock::system_time}; + current_clock.wall_time = {&CMOS_clock::wall_clock}; + current_clock.tsc_khz = {&CMOS_clock::get_tsc_khz}; CMOS_clock::init(); INFO("x86", "CMOS clock initialized"); } @@ -66,15 +69,15 @@ namespace x86 KHz Clocks::get_khz() { - return PER_CPU(vcpu_clock).tsc_khz(); + return current_clock.tsc_khz(); } } uint64_t __arch_system_time() noexcept { - return PER_CPU(vcpu_clock).system_time(); + return current_clock.system_time(); } timespec __arch_wall_clock() noexcept { - return PER_CPU(vcpu_clock).wall_time(); + return current_clock.wall_time(); } From a32db027e0723877e24e9afad91ec573d852f63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Mon, 2 Jul 2018 14:53:38 +0200 Subject: [PATCH 480/723] liveupdate: Align virtual area to pages --- lib/LiveUpdate/os.cpp | 3 ++- src/plugins/system_log.cpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/LiveUpdate/os.cpp b/lib/LiveUpdate/os.cpp index c641a4e90e..1141c1a4ca 100644 --- a/lib/LiveUpdate/os.cpp +++ b/lib/LiveUpdate/os.cpp @@ -29,11 +29,12 @@ void OS::setup_liveupdate(uintptr_t phys) if (phys == 0) { // default is 1/4 of heap from the end of memory size = OS::heap_max() / 4; - phys = (OS::heap_max() - size) & 0xFFFFFFF0; + phys = (OS::heap_max() - size) & ~(uintptr_t) 0xFFF; // page aligned } else { size = OS::heap_max() - phys; } + size &= ~(uintptr_t) 0xFFF; // page sized // move location to high memory const uintptr_t dst = (uintptr_t) OS::liveupdate_storage_area(); diff --git a/src/plugins/system_log.cpp b/src/plugins/system_log.cpp index 9619ec307b..acc07df5bc 100644 --- a/src/plugins/system_log.cpp +++ b/src/plugins/system_log.cpp @@ -29,7 +29,7 @@ static inline RingBuffer* get_mrb() inline static char* get_system_log_loc() { #ifdef ARCH_x86_64 - return (char*) ((1ull << 46) - MRB_AREA_SIZE); + return (char*) ((1ull << 45) - MRB_AREA_SIZE); #else return (char*) OS::liveupdate_storage_area() - MRB_AREA_SIZE; #endif From 101a6b743905f2fc95fb36721c51e39151de7624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Mon, 2 Jul 2018 14:54:09 +0200 Subject: [PATCH 481/723] x86: Minor improvements to ACPI shutdown --- src/kernel/os.cpp | 1 - src/platform/x86_pc/acpi.cpp | 22 +++++++++------------- src/platform/x86_pc/acpi.hpp | 8 ++++---- 3 files changed, 13 insertions(+), 18 deletions(-) diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index ea5cb60ddd..e8e9c9691f 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -119,7 +119,6 @@ void OS::reboot() } void OS::shutdown() { - MYINFO("Soft shutdown signalled"); power_ = false; } diff --git a/src/platform/x86_pc/acpi.cpp b/src/platform/x86_pc/acpi.cpp index c1871028c9..5255391c63 100644 --- a/src/platform/x86_pc/acpi.cpp +++ b/src/platform/x86_pc/acpi.cpp @@ -22,6 +22,7 @@ #include extern "C" void reboot_os(); +extern "C" void apm_shutdown(); namespace x86 { @@ -311,8 +312,8 @@ namespace x86 { ACPI_ENABLE = facp->ACPI_ENABLE; ACPI_DISABLE = facp->ACPI_DISABLE; - PM1a_CNT = (uint32_t*) (uintptr_t) facp->PM1a_CNT_BLK; - PM1b_CNT = (uint32_t*) (uintptr_t) facp->PM1b_CNT_BLK; + PM1a_CNT = facp->PM1a_CNT_BLK; + PM1b_CNT = facp->PM1b_CNT_BLK; PM1_CNT_LEN = facp->PM1_CNT_LEN; @@ -386,10 +387,9 @@ namespace x86 { // check if shutdown enabled if (SCI_EN == 1) { // write shutdown commands - hw::outw((uintptr_t) PM1a_CNT, SLP_TYPa | SLP_EN); + hw::outw(PM1a_CNT, SLP_TYPa | SLP_EN); if (PM1b_CNT != 0) - hw::outw((uintptr_t) PM1b_CNT, SLP_TYPb | SLP_EN); - printf("*** ACPI shutdown failed\n"); + hw::outw(PM1b_CNT, SLP_TYPb | SLP_EN); } } @@ -401,17 +401,13 @@ namespace x86 { // ACPI shutdown get().acpi_shutdown(); - // http://forum.osdev.org/viewtopic.php?t=16990 hw::outw (0xB004, 0x2000); - const char s[] = "Shutdown"; - const char *p; - for (p = s; *p; p++) - // magic code for bochs and qemu - hw::outb (0x8900, *s); + // magic code for bochs and qemu + const char* s = "Shutdown"; + while (*s) hw::outb (0x8900, *(s++)); // VMWare poweroff when "gui.exitOnCLIHLT" is true - printf("Shutdown failed :(\n"); while (true) asm ("cli; hlt"); - } + } // shutdown() } diff --git a/src/platform/x86_pc/acpi.hpp b/src/platform/x86_pc/acpi.hpp index 97b3b72f78..118b371094 100644 --- a/src/platform/x86_pc/acpi.hpp +++ b/src/platform/x86_pc/acpi.hpp @@ -116,10 +116,10 @@ namespace x86 { void acpi_shutdown(); uint32_t* SMI_CMD; - uint8_t ACPI_ENABLE; - uint8_t ACPI_DISABLE; - uint32_t* PM1a_CNT; - uint32_t* PM1b_CNT; + uint8_t ACPI_ENABLE; + uint8_t ACPI_DISABLE; + uint32_t PM1a_CNT; + uint32_t PM1b_CNT; uint16_t SLP_TYPa; uint16_t SLP_TYPb; uint16_t SLP_EN; From 8177add5a61b9bbf9cdc723528035b9b37bf39f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 3 Jul 2018 10:30:08 +0200 Subject: [PATCH 482/723] uplink: Install headers and make instance retreivable from service --- lib/uplink/CMakeLists.txt | 7 ++++++ lib/uplink/config.hpp | 2 +- lib/uplink/log.hpp | 2 +- lib/uplink/register_plugin.cpp | 30 ++++++++---------------- lib/uplink/uplink.cpp | 43 ++++++++++++++++++++++++++++++++++ lib/uplink/uplink.hpp | 25 ++++++++++++++++++++ 6 files changed, 87 insertions(+), 22 deletions(-) create mode 100644 lib/uplink/uplink.cpp create mode 100644 lib/uplink/uplink.hpp diff --git a/lib/uplink/CMakeLists.txt b/lib/uplink/CMakeLists.txt index 46b3f6373f..29a2693d70 100644 --- a/lib/uplink/CMakeLists.txt +++ b/lib/uplink/CMakeLists.txt @@ -23,8 +23,15 @@ set(SOURCES ws_uplink.cpp register_plugin.cpp config.cpp + uplink.cpp ) +# Install headers +install(DIRECTORY . DESTINATION includeos/include/uplink + FILES_MATCHING PATTERN "*.hpp" + PATTERN "starbase" EXCLUDE + PATTERN "build" EXCLUDE) + # Uplink library add_library(${LIBRARY_NAME} STATIC ${SOURCES}) add_dependencies(${LIBRARY_NAME} PrecompiledLibraries) diff --git a/lib/uplink/config.hpp b/lib/uplink/config.hpp index 897659279f..3be19041dd 100644 --- a/lib/uplink/config.hpp +++ b/lib/uplink/config.hpp @@ -20,7 +20,7 @@ #define UPLINK_CONFIG_HPP #include -#include +#include #include namespace uplink { diff --git a/lib/uplink/log.hpp b/lib/uplink/log.hpp index 35f625bca8..fa8b29afd8 100644 --- a/lib/uplink/log.hpp +++ b/lib/uplink/log.hpp @@ -21,8 +21,8 @@ #include #include +#include #include -#include namespace uplink { diff --git a/lib/uplink/register_plugin.cpp b/lib/uplink/register_plugin.cpp index d2d756d858..1f4cc53a90 100644 --- a/lib/uplink/register_plugin.cpp +++ b/lib/uplink/register_plugin.cpp @@ -15,34 +15,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "ws_uplink.hpp" +#include "uplink.hpp" #include "common.hpp" -#include - -namespace uplink { - -static std::unique_ptr uplink{nullptr}; +#include -void setup_uplink() +static void setup_uplink_plugin() { - MYINFO("Setting up WS uplink"); - - try { - auto config = Config::read(); - - uplink = std::make_unique(std::move(config)); - - }catch(const std::exception& e) { - MYINFO("Uplink initialization failed: %s ", e.what()); + try + { + uplink::get(); + } + catch(const std::exception& e) + { MYINFO("Rebooting"); OS::reboot(); } } -} // < namespace uplink - -#include __attribute__((constructor)) void register_plugin_uplink(){ - OS::register_plugin(uplink::setup_uplink, "Uplink"); + OS::register_plugin(setup_uplink_plugin, "Uplink"); } diff --git a/lib/uplink/uplink.cpp b/lib/uplink/uplink.cpp new file mode 100644 index 0000000000..e60a697b27 --- /dev/null +++ b/lib/uplink/uplink.cpp @@ -0,0 +1,43 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "uplink.hpp" +#include "common.hpp" + +namespace uplink { + + static WS_uplink setup_uplink() + { + MYINFO("Setting up WS uplink"); + try + { + auto config = Config::read(); + return WS_uplink{std::move(config)}; + } + catch(const std::exception& e) + { + MYINFO("Uplink initialization failed: %s ", e.what()); + throw; + } + } + + WS_uplink& get() + { + static WS_uplink instance{setup_uplink()}; + return instance; + } + +} diff --git a/lib/uplink/uplink.hpp b/lib/uplink/uplink.hpp new file mode 100644 index 0000000000..e80d965935 --- /dev/null +++ b/lib/uplink/uplink.hpp @@ -0,0 +1,25 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "ws_uplink.hpp" + +namespace uplink { + + WS_uplink& get(); + +} // < namespace uplink From 659dfe740efe69dd2a18b5275fb0585955bbb84d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Jul 2018 14:11:14 +0200 Subject: [PATCH 483/723] linux: Update to latest dev --- linux/src/main.cpp | 2 +- linux/src/os.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/linux/src/main.cpp b/linux/src/main.cpp index 8f0f34aac2..4fc0ddfd93 100644 --- a/linux/src/main.cpp +++ b/linux/src/main.cpp @@ -38,7 +38,7 @@ int main(int, char** args) #endif // initialize Linux platform - OS::start(args[0], 0u); + OS::start(args[0]); // calls Service::start OS::post_start(); diff --git a/linux/src/os.cpp b/linux/src/os.cpp index e93c39206d..8708ff710c 100644 --- a/linux/src/os.cpp +++ b/linux/src/os.cpp @@ -81,7 +81,7 @@ static void begin_timer(std::chrono::nanoseconds usec) static void stop_timers() {} #include -void OS::start(char* cmdline, uintptr_t) +void OS::start(const char* cmdline) { __libc_initialized = true; // statman From 13405d631e1f20245d4df6244b89cabe9cc7cbbe Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 3 Jul 2018 14:11:54 +0200 Subject: [PATCH 484/723] util: Serialize/deserialize statman --- api/util/statman.hpp | 8 +++++ lib/uplink/ws_uplink.cpp | 21 +++++++++++++ src/CMakeLists.txt | 4 ++- src/util/statman.cpp | 9 ++++++ src/util/statman_liu.cpp | 30 +++++++++++++++++++ .../integration/LiveUpdate/test_boot.cpp | 20 +++++++++++++ 6 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 src/util/statman_liu.cpp diff --git a/api/util/statman.hpp b/api/util/statman.hpp index fb4df08aef..6f162e4dfe 100644 --- a/api/util/statman.hpp +++ b/api/util/statman.hpp @@ -35,6 +35,10 @@ struct Stats_exception : public std::runtime_error { using runtime_error::runtime_error; }; +namespace liu { + struct Storage; struct Restore; +} + class Stat { public: static const int MAX_NAME_LEN = 46; @@ -74,6 +78,8 @@ class Stat { /// uint64_t& get_uint64(); + std::string to_string() const; + private: union { float f; @@ -169,6 +175,8 @@ class Statman { Stat* begin() noexcept { return (Stat*) cbegin(); } Stat* end() noexcept { return (Stat*) cend(); } + void store(uint32_t id, liu::Storage&); + void restore(liu::Restore&); void init(const uintptr_t location, const Size_type size); diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index e75caa3335..f06af56672 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -142,6 +142,20 @@ namespace uplink { store.add_string(0, update_hash_); // nanos timestamp of when update begins store.add (1, OS::nanos_since_boot()); + // statman + auto& stm = Statman::get(); + // increment number of updates performed + try { + ++stm.get_by_name("system.updates"); + } + catch (std::exception e) + { + ++stm.create(Stat::UINT32, "system.updates"); + } + // store all stats + stm.store(2, store); + // go to end + store.put_marker(100); } void WS_uplink::restore(liu::Restore& store) @@ -152,6 +166,13 @@ namespace uplink { // calculate update cycles taken uint64_t prev_nanos = store.as_type (); store.go_next(); this->update_time_taken = OS::nanos_since_boot() - prev_nanos; + // statman + if (!store.is_end()) + { + Statman::get().restore(store); + } + // done marker + store.pop_marker(100); INFO2("Update took %.3f millis", this->update_time_taken / 1.0e6); } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b6bad93cea..8f9ae4fd6b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,6 +16,7 @@ include_directories(${INCLUDEOS_ROOT}/mod/) include_directories(${INCLUDEOS_ROOT}/mod/GSL/) include_directories(${INCLUDEOS_ROOT}/mod/rapidjson/include) include_directories(${INCLUDEOS_ROOT}/mod/uzlib/src) # tinf.h for tar +include_directories(${INCLUDEOS_ROOT}/lib/LiveUpdate) include_directories(${BOTAN_DIR}) include_directories(${OPENSSL_DIR}/include) if(${ARCH} STREQUAL "x86_64") @@ -35,7 +36,8 @@ set(OS_OBJECTS kernel/fiber.cpp kernel/tls.cpp kernel/profile.cpp kernel/scoped_profiler.cpp kernel/terminal.cpp kernel/timers.cpp kernel/rtc.cpp kernel/rng.cpp kernel/system_log.cpp kernel/rdrand.cpp kernel/solo5_manager.cpp - util/memstream.c util/async.cpp util/statman.cpp util/logger.cpp util/sha1.cpp + util/memstream.c util/async.cpp util/statman.cpp "util/statman_liu.cpp" + util/logger.cpp util/sha1.cpp util/syslog_facility.cpp util/syslogd.cpp util/uri.cpp util/percent_encoding.cpp util/tar.cpp util/path_to_regex.cpp util/config.cpp util/autoconf.cpp util/crc32.cpp crt/c_abi.c crt/ctype_b_loc.c crt/ctype_tolower_loc.c crt/string.c diff --git a/src/util/statman.cpp b/src/util/statman.cpp index c7dd191102..ad4b761a2d 100644 --- a/src/util/statman.cpp +++ b/src/util/statman.cpp @@ -66,6 +66,15 @@ uint64_t& Stat::get_uint64() { return ui64; } +std::string Stat::to_string() const { + switch (this->type_) { + case UINT32: return std::to_string(ui32); + case UINT64: return std::to_string(ui64); + case FLOAT: return std::to_string(f); + default: return "Invalid Stat_type"; + } +} + /////////////////////////////////////////////////////////////////////////////// diff --git a/src/util/statman_liu.cpp b/src/util/statman_liu.cpp new file mode 100644 index 0000000000..2c5d5cc062 --- /dev/null +++ b/src/util/statman_liu.cpp @@ -0,0 +1,30 @@ +#include +#include + +void Statman::store(uint32_t id, liu::Storage& store) +{ + store.add_buffer(id, this->cbegin(), this->num_bytes()); +} +void Statman::restore(liu::Restore& store) +{ + auto buffer = store.as_buffer(); store.go_next(); + + assert(buffer.size() % sizeof(Stat) == 0); + const size_t count = buffer.size() / sizeof(Stat); + + const Stat* ptr = (Stat*) buffer.data(); + const Stat* end = ptr + count; + + for (; ptr < end; ptr++) + { + try { + auto& stat = this->get_by_name(ptr->name()); + std::memcpy(&stat, ptr, sizeof(Stat)); + } + catch (std::exception e) + { + auto& stat = this->create(ptr->type(), ptr->name()); + std::memcpy(&stat, ptr, sizeof(Stat)); + } + } +} diff --git a/test/kernel/integration/LiveUpdate/test_boot.cpp b/test/kernel/integration/LiveUpdate/test_boot.cpp index 47d633308b..b6ff805663 100644 --- a/test/kernel/integration/LiveUpdate/test_boot.cpp +++ b/test/kernel/integration/LiveUpdate/test_boot.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include using namespace liu; @@ -28,6 +29,16 @@ static void boot_save(Storage& storage, const buffer_t* blob) #else storage.add_buffer(2, *blob); #endif + auto& stm = Statman::get(); + // increment number of updates performed + try { + ++stm.get_by_name("system.updates"); + } + catch (std::exception e) + { + ++stm.create(Stat::UINT32, "system.updates"); + } + stm.store(3, storage); } static void boot_resume_all(Restore& thing) { @@ -46,6 +57,11 @@ static void boot_resume_all(Restore& thing) #else bloberino = thing.as_buffer(); thing.go_next(); #endif + // statman + auto& stm = Statman::get(); + stm.restore(thing); + auto& stat = stm.get_by_name("system.updates"); + assert(stat.get_uint32() > 0); thing.pop_marker(); } @@ -69,6 +85,10 @@ LiveUpdate::storage_func begin_test_boot() printf("%lld\n", stamp); } */ + for (auto& stat : Statman::get()) { + printf("%s: %s\n", stat.name(), stat.to_string().c_str()); + } + printf("Verifying that timers are started...\n"); using namespace std::chrono; From 21946976727915655b94bef2220894e658b22128 Mon Sep 17 00:00:00 2001 From: taiyeba Date: Wed, 4 Jul 2018 10:43:06 +0200 Subject: [PATCH 485/723] test: fixing parallel build tests --- test/misc/build_examples/test.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/misc/build_examples/test.sh b/test/misc/build_examples/test.sh index 3e9cef0100..7dc22e6165 100755 --- a/test/misc/build_examples/test.sh +++ b/test/misc/build_examples/test.sh @@ -62,7 +62,9 @@ do continue fi # build_service "$dir" - parallel build_service ::: "$dir" + # parallel build_service ::: "$dir" + # build_service "$dir" | xargs -n 8 + build_service "$dir" | xargs done echo "Done" From 10e3f5fb7d4957d78d586b41cd5d622151e3a1b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 5 Jul 2018 11:55:06 +0200 Subject: [PATCH 486/723] net: Let DHCP continue forever if no timeout is given #1716 --- api/net/dhcp/dh4client.hpp | 7 +++--- src/net/dhcp/dh4client.cpp | 49 ++++++++++++++++++++++++++++++-------- src/net/inet.cpp | 2 +- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/api/net/dhcp/dh4client.hpp b/api/net/dhcp/dh4client.hpp index 320fa5263f..420b1c2ca1 100644 --- a/api/net/dhcp/dh4client.hpp +++ b/api/net/dhcp/dh4client.hpp @@ -30,7 +30,8 @@ namespace net { class DHClient { public: - static const int NUM_RETRIES = 5; + static constexpr std::chrono::seconds RETRY_FREQUENCY{1}; + static constexpr std::chrono::seconds RETRY_FREQUENCY_SLOW{10}; using Stack = IP4::Stack; using config_func = delegate; @@ -40,7 +41,7 @@ namespace net { DHClient(Stack& inet); // negotiate with local DHCP server - void negotiate(uint32_t timeout_secs); + void negotiate(std::chrono::seconds timeout = std::chrono::seconds::zero()); // Signal indicating the result of DHCP negotation // timeout is true if the negotiation timed out @@ -61,7 +62,7 @@ namespace net { std::string domain_name; uint32_t lease_time; std::vector config_handlers_; - int retries = 0; + int tries = 0; int progress = 0; Timer timeout_timer_; std::chrono::milliseconds timeout; diff --git a/src/net/dhcp/dh4client.cpp b/src/net/dhcp/dh4client.cpp index 52fefbabf5..8694656e3c 100644 --- a/src/net/dhcp/dh4client.cpp +++ b/src/net/dhcp/dh4client.cpp @@ -34,7 +34,6 @@ namespace net { using namespace dhcp; - const int DHClient::NUM_RETRIES; DHClient::DHClient(Stack& inet) : stack(inet), @@ -60,24 +59,54 @@ namespace net { void DHClient::restart_negotation() { - if (retries-- <= 0) + tries++; + // if timeout is supplied + if(timeout != std::chrono::seconds::zero()) { - // give up when retries reached zero - end_negotiation(true); + // calculate if we should retry + const bool retry = (timeout / (tries * RETRY_FREQUENCY)) >= 1; + + if(retry) + { + timeout_timer_.start(RETRY_FREQUENCY); + send_first(); + return; + } + else + { + end_negotiation(true); + return; + } + } + + static const int FAST_TRIES = 10; + // never timeout + if(tries <= FAST_TRIES) + { + // do fast retry + timeout_timer_.start(RETRY_FREQUENCY); } else { - timeout_timer_.start(this->timeout); - send_first(); + if(UNLIKELY(tries == FAST_TRIES+1)) { + MYINFO("No reply for %i tries, retrying every %lli second", + FAST_TRIES, RETRY_FREQUENCY_SLOW.count()); + } + + // fallback to slow retry + timeout_timer_.start(RETRY_FREQUENCY_SLOW); } + + send_first(); } + void DHClient::end_negotiation(bool timed_out) { // wind down this->xid = 0; timeout_timer_.stop(); this->progress = 0; - this->retries = NUM_RETRIES; + this->tries = 0; // close UDP socket Expects(this->socket != nullptr); this->socket->close(); @@ -89,16 +118,16 @@ namespace net { if(timeout_timer_.is_running()) timeout_timer_.stop(); } - void DHClient::negotiate(uint32_t timeout_secs) + void DHClient::negotiate(std::chrono::seconds timeout) { // Allow multiple calls to negotiate without restarting the process if (this->xid != 0) return; - this->retries = NUM_RETRIES; + this->tries = 0; this->progress = 0; // calculate progress timeout using namespace std::chrono; - this->timeout = seconds(timeout_secs) / NUM_RETRIES; + this->timeout = timeout; // generate a new session ID this->xid = (rand() & 0xffff); diff --git a/src/net/inet.cpp b/src/net/inet.cpp index 9b3b924158..5044c935bb 100644 --- a/src/net/inet.cpp +++ b/src/net/inet.cpp @@ -203,7 +203,7 @@ void Inet::negotiate_dhcp(double timeout, dhcp_timeout_func handler) { if (!dhcp_) dhcp_ = std::make_shared(*this); // @timeout for DHCP-server negotation - dhcp_->negotiate(timeout); + dhcp_->negotiate(std::chrono::seconds((uint32_t)timeout)); // add timeout_handler if supplied if (handler) dhcp_->on_config(handler); From 92c002ea65baf6932827388ba33ef0cc731cadca Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Fri, 13 Jul 2018 13:28:43 +0200 Subject: [PATCH 487/723] Updating to latest NaCl (Timer class added) --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index 7b7490c4aa..5bc2ab0634 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit 7b7490c4aa1597c832907aa934bcea0f25e29b3c +Subproject commit 5bc2ab0634735177607f68af023b99800f9fdaa2 From 1364d68165324a7d9793b6e3c04a7e5c0010e8af Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 13 Jul 2018 14:41:54 +0200 Subject: [PATCH 488/723] packet: Rewrite chain() and prevent loops --- api/net/packet.hpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/api/net/packet.hpp b/api/net/packet.hpp index f09865d507..b8f23858a3 100644 --- a/api/net/packet.hpp +++ b/api/net/packet.hpp @@ -190,8 +190,19 @@ namespace net Byte buf_[0]; }; //< class Packet - void Packet::chain(Packet_ptr p) noexcept + void Packet::chain(Packet_ptr pkt) noexcept { + assert(pkt.get() != nullptr); + assert(pkt.get() != this); + + auto* p = this; + while (p->chain_ != nullptr) { + p = p->chain_.get(); + assert(pkt.get() != p); + } + p->chain_ = std::move(pkt); + + /* if (!chain_) { chain_ = std::move(p); last_ = chain_.get(); @@ -201,6 +212,7 @@ namespace net last_ = ptr->last_in_chain() ? ptr->last_in_chain() : ptr; assert(last_); } + */ } int Packet::chain_length() const noexcept From a61224c66ad487c1a17af8a85e9e6fa45c24d84f Mon Sep 17 00:00:00 2001 From: John Zhang Date: Fri, 20 Jul 2018 22:48:13 +0800 Subject: [PATCH 489/723] catch exception by const reference --- lib/mender/include/mender/inventory.hpp | 2 +- lib/uplink/ws_uplink.cpp | 4 ++-- src/util/statman_liu.cpp | 2 +- test/kernel/integration/LiveUpdate/test_boot.cpp | 2 +- test/stl/integration/exceptions/service.cpp | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/mender/include/mender/inventory.hpp b/lib/mender/include/mender/inventory.hpp index c36305f8e1..d098d064dc 100644 --- a/lib/mender/include/mender/inventory.hpp +++ b/lib/mender/include/mender/inventory.hpp @@ -80,7 +80,7 @@ namespace mender { try { return data_.at(key); } - catch(std::out_of_range) { + catch(const std::out_of_range&) { return {}; } } diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index 5736b2d12d..d0c0439752 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -148,7 +148,7 @@ namespace uplink { try { ++stm.get_by_name("system.updates"); } - catch (std::exception e) + catch (const std::exception& e) { ++stm.create(Stat::UINT32, "system.updates"); } @@ -409,7 +409,7 @@ namespace uplink { try { liu::LiveUpdate::exec(std::move(buffer)); } - catch (std::exception& e) { + catch (const std::exception& e) { INFO2("LiveUpdate::exec() failed: %s\n", e.what()); liu::LiveUpdate::restore_environment(); // establish new connection diff --git a/src/util/statman_liu.cpp b/src/util/statman_liu.cpp index 2c5d5cc062..bac4a8ba58 100644 --- a/src/util/statman_liu.cpp +++ b/src/util/statman_liu.cpp @@ -21,7 +21,7 @@ void Statman::restore(liu::Restore& store) auto& stat = this->get_by_name(ptr->name()); std::memcpy(&stat, ptr, sizeof(Stat)); } - catch (std::exception e) + catch (const std::exception& e) { auto& stat = this->create(ptr->type(), ptr->name()); std::memcpy(&stat, ptr, sizeof(Stat)); diff --git a/test/kernel/integration/LiveUpdate/test_boot.cpp b/test/kernel/integration/LiveUpdate/test_boot.cpp index b6ff805663..ea974ad41e 100644 --- a/test/kernel/integration/LiveUpdate/test_boot.cpp +++ b/test/kernel/integration/LiveUpdate/test_boot.cpp @@ -34,7 +34,7 @@ static void boot_save(Storage& storage, const buffer_t* blob) try { ++stm.get_by_name("system.updates"); } - catch (std::exception e) + catch (const std::exception& e) { ++stm.create(Stat::UINT32, "system.updates"); } diff --git a/test/stl/integration/exceptions/service.cpp b/test/stl/integration/exceptions/service.cpp index e4e0aaf331..dd65e4d620 100644 --- a/test/stl/integration/exceptions/service.cpp +++ b/test/stl/integration/exceptions/service.cpp @@ -41,7 +41,7 @@ const lest::test tests[] = { std::runtime_error myexception(error_msg); throw myexception; } - } catch(std::runtime_error e) { + } catch(const std::runtime_error& e) { caught = std::string(e.what()) == std::string(error_msg); } EXPECT(caught); @@ -53,7 +53,7 @@ const lest::test tests[] = { try { throw CustomException(custom_msg); - } catch (CustomException e){ + } catch (const CustomException& e){ caught_msg = e.what(); } From 8a734b886aeb389ecd60be8d5f26ac315ef37b96 Mon Sep 17 00:00:00 2001 From: Ricardo Koller Date: Mon, 23 Jul 2018 17:22:11 -0400 Subject: [PATCH 490/723] Point to solo5 v0.3.0 This is mainly for #1515 as solo5 version (0.3.0) has a fix for it. The solo5 interface changed slightly for 0.3.0: function names, and args to `solo5_app_main` and `solo5_block_read`. --- cmake/cross_compiled_libraries.cmake | 2 +- src/drivers/solo5blk.cpp | 38 ++++++++++++++++++------- src/drivers/solo5net.cpp | 11 ++++--- src/platform/x86_solo5/kernel_start.cpp | 29 +++++++------------ src/platform/x86_solo5/os.cpp | 4 +-- 5 files changed, 49 insertions(+), 35 deletions(-) diff --git a/cmake/cross_compiled_libraries.cmake b/cmake/cross_compiled_libraries.cmake index 74cb21c669..9f9da88b55 100644 --- a/cmake/cross_compiled_libraries.cmake +++ b/cmake/cross_compiled_libraries.cmake @@ -33,7 +33,7 @@ ExternalProject_Add(solo5_repo PREFIX precompiled BUILD_IN_SOURCE 1 GIT_REPOSITORY https://github.com/solo5/solo5.git - GIT_TAG 2765e0f5f090c0b27a8d62a48285842236e7d20f + GIT_TAG v0.3.0 CONFIGURE_COMMAND CC=gcc ./configure.sh UPDATE_COMMAND "" BUILD_COMMAND make diff --git a/src/drivers/solo5blk.cpp b/src/drivers/solo5blk.cpp index aad5a24d42..b6edf49207 100644 --- a/src/drivers/solo5blk.cpp +++ b/src/drivers/solo5blk.cpp @@ -16,27 +16,45 @@ extern "C" { Solo5Blk::Solo5Blk() : hw::Block_device() { - INFO("Solo5Blk", "Block device with %zu sectors", solo5_blk_sectors()); + struct solo5_block_info bi; + solo5_block_info(&bi); + assert(bi.block_size == SECTOR_SIZE); + INFO("Solo5Blk", "Block device with %zu sectors", + bi.capacity / SECTOR_SIZE); } Solo5Blk::block_t Solo5Blk::size() const noexcept { - return solo5_blk_sectors(); + struct solo5_block_info bi; + solo5_block_info(&bi); + return bi.capacity / SECTOR_SIZE; } -Solo5Blk::buffer_t Solo5Blk::read_sync(block_t blk) -{ - auto buffer = fs::construct_buffer(block_size()); - int rlen = SECTOR_SIZE; +Solo5Blk::buffer_t Solo5Blk::read_sync(block_t blk) { + auto buffer = fs::construct_buffer(SECTOR_SIZE); + solo5_result_t res; - solo5_blk_read_sync((uint64_t) blk, buffer->data(), &rlen); + res = solo5_block_read((solo5_off_t) blk * SECTOR_SIZE, + buffer->data(), SECTOR_SIZE); + if (res != SOLO5_R_OK) { + buffer.reset(); + return nullptr; + } return buffer; } Solo5Blk::buffer_t Solo5Blk::read_sync(block_t blk, size_t count) { - auto buffer = fs::construct_buffer(block_size() * count); - int rlen = SECTOR_SIZE * count; + auto buffer = fs::construct_buffer(SECTOR_SIZE * count); + solo5_result_t res; - solo5_blk_read_sync((uint64_t) blk, buffer->data(), &rlen); + auto* data = (uint8_t*) buffer->data(); + for (size_t i = 0; i < count; i++) { + res = solo5_block_read((solo5_off_t) (blk + i) * SECTOR_SIZE, + data + (i * SECTOR_SIZE), SECTOR_SIZE); + if (res != SOLO5_R_OK) { + buffer.reset(); + return nullptr; + } + } return buffer; } diff --git a/src/drivers/solo5net.cpp b/src/drivers/solo5net.cpp index 3b564f2b73..68dc032356 100644 --- a/src/drivers/solo5net.cpp +++ b/src/drivers/solo5net.cpp @@ -36,7 +36,10 @@ Solo5Net::Solo5Net() bufstore_{NUM_BUFFERS, 2048u} // don't change this { INFO("Solo5Net", "Driver initializing"); - mac_addr = MAC::Addr(solo5_net_mac_str()); + struct solo5_net_info ni; + solo5_net_info(&ni); + mac_addr = MAC::Addr(ni.mac_address[0], ni.mac_address[1], ni.mac_address[2], + ni.mac_address[3], ni.mac_address[4], ni.mac_address[5]); } void Solo5Net::transmit(net::Packet_ptr pckt) @@ -48,7 +51,7 @@ void Solo5Net::transmit(net::Packet_ptr pckt) // next in line auto next = tail->detach_tail(); // write data to network - solo5_net_write_sync(tail->buf(), tail->size()); + solo5_net_write(tail->buf(), tail->size()); // set tail to next, releasing tail tail = std::move(next); // Stat increase packets transmitted @@ -75,8 +78,8 @@ net::Packet_ptr Solo5Net::recv_packet() auto* pckt = (net::Packet*) buffer.addr; new (pckt) net::Packet(0, MTU(), packet_len(), buffer.bufstore); // Populate the packet buffer with new packet, if any - int size = packet_len(); - if (solo5_net_read_sync(pckt->buf(), &size) == 0) { + size_t size = packet_len(); + if (solo5_net_read(pckt->buf(), size, &size) == 0) { // Adjust packet size to match received data if (size) { pckt->set_data_end(size); diff --git a/src/platform/x86_solo5/kernel_start.cpp b/src/platform/x86_solo5/kernel_start.cpp index 9f2b72a87b..778c736194 100644 --- a/src/platform/x86_solo5/kernel_start.cpp +++ b/src/platform/x86_solo5/kernel_start.cpp @@ -26,6 +26,7 @@ extern "C" { static char temp_cmdline[1024]; static uintptr_t mem_size = 0; +static uintptr_t free_mem_begin; extern "C" void kernel_start() @@ -33,9 +34,6 @@ void kernel_start() // generate checksums of read-only areas etc. __init_sanity_checks(); - // Determine where free memory starts - uintptr_t free_mem_begin = reinterpret_cast(&_end); - // Preserve symbols from the ELF binary free_mem_begin += _move_symbols(free_mem_begin); @@ -43,9 +41,6 @@ void kernel_start() //_init_bss(); // Initialize heap - // XXX: this is dangerous as solo5 might be doing malloc()'s using it's own - // idea of a heap. Luckily there is no malloc instance at solo5/kernel/[ukvm|virtio|muen], - // so might be OK (for now). OS::init_heap(free_mem_begin, mem_size); _init_elf_parser(); @@ -62,17 +57,15 @@ void kernel_start() } extern "C" -int solo5_app_main(char* cmdline) +int solo5_app_main(const struct solo5_start_info *si) { - // cmdline is stored at 0x6000 by ukvm which is used by includeos. Move it fast. - strncpy(temp_cmdline, cmdline, sizeof(temp_cmdline)-1); - temp_cmdline[sizeof(temp_cmdline)-1] = 0; - - // solo5 sets the stack to be at the end of memory, so let's use that as - // our memory size (before we change). - mem_size = (uintptr_t) get_cpu_ebp(); - - // set the stack location to its new includeos location, and call kernel_start - set_stack(); - return 0; + // si is stored at 0x6000 by ukvm which is used by includeos. Move it fast. + strncpy(temp_cmdline, si->cmdline, sizeof(temp_cmdline)-1); + temp_cmdline[sizeof(temp_cmdline)-1] = 0; + free_mem_begin = si->heap_start; + mem_size = si->heap_size; + + // set the stack location to its new includeos location, and call kernel_start + set_stack(); + return 0; } diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index dabe9d2c06..0740e1f79d 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -154,7 +154,7 @@ void OS::event_loop() // XXX: temporarily ALWAYS sleep for 0.5 ms. We should ideally ask Timers // for the next immediate timer to fire (the first from the "scheduled" list // of timers?) - rc = solo5_poll(solo5_clock_monotonic() + 500000ULL); // now + 0.5 ms + rc = solo5_yield(solo5_clock_monotonic() + 500000ULL); // now + 0.5 ms Timers::timers_handler(); if (rc) { for(auto& nic : hw::Devices::devices()) { @@ -198,7 +198,7 @@ void OS::block() highest_blocking_level = blocking_level; int rc; - rc = solo5_poll(solo5_clock_monotonic() + 50000ULL); // now + 0.05 ms + rc = solo5_yield(solo5_clock_monotonic() + 50000ULL); // now + 0.05 ms if (rc == 0) { Timers::timers_handler(); } else { From c98001948b6d8fdad82edc2142437963da25c6e8 Mon Sep 17 00:00:00 2001 From: Ricardo Koller Date: Wed, 25 Jul 2018 14:05:51 -0400 Subject: [PATCH 491/723] Adding boot and vmrunner support for solo5 disks --- etc/boot | 1 - vmrunner/vmrunner.py | 19 ++++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/etc/boot b/etc/boot index e5b8bd442c..bea70193dd 100755 --- a/etc/boot +++ b/etc/boot @@ -162,7 +162,6 @@ if args.solo5: os.environ['PLATFORM'] = "x86_solo5" qkvm_bin = INCLUDEOS_PREFIX + "/includeos/x86_64/lib/ukvm-bin" - subprocess.call(['touch', INCLUDEOS_PREFIX + 'dummy.disk']) subprocess.call(['chmod', '+x', qkvm_bin]) subprocess.call(['sudo', INCLUDEOS_PREFIX + "/includeos/scripts/ukvm-ifup.sh" ]) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 2518e2414d..87d2e372ed 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -215,8 +215,13 @@ def name(self): def image_name(self): return self._image_name - def drive_arg(self): - return ["--disk=" + INCLUDEOS_HOME + "dummy.disk"] + def drive_arg(self, filename, + device_format="raw", media_type="disk"): + if device_format != "raw": + raise Exception("solo5/ukvm can only handle drives in raw format.") + if media_type != "disk": + raise Exception("solo5/ukvm can only handle drives of type disk.") + return ["--disk=" + filename] def net_arg(self): return ["--net=tap100"] @@ -236,7 +241,15 @@ def boot(self, multiboot, debug=False, kernel_args = "", image_name = None): self._image_name = image_name command = ["sudo", qkvm_bin] - command += self.drive_arg() + + if "drives" in self._config: + if len(self._config["drives"]) > 1: + raise Exception("solo5/ukvm can only handle one drive.") + for disk in self._config["drives"]: + info ("Ignoring drive type argument: ", disk["type"]) + command += self.drive_arg(disk["file"], disk["format"], + disk["media"]) + command += self.net_arg() command += [self._image_name] command += [kernel_args] From f3199bd3aaa4ef572598c415486c5f68efd3ad60 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 30 Jul 2018 12:22:39 +0200 Subject: [PATCH 492/723] vmrunner: Support solo5 instances without disks --- vmrunner/vmrunner.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 87d2e372ed..41e04730af 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -242,9 +242,11 @@ def boot(self, multiboot, debug=False, kernel_args = "", image_name = None): command = ["sudo", qkvm_bin] - if "drives" in self._config: - if len(self._config["drives"]) > 1: - raise Exception("solo5/ukvm can only handle one drive.") + if not "drives" in self._config: + command += self.drive_arg(self._image_name) + elif len(self._config["drives"]) > 1: + raise Exception("solo5/ukvm can only handle one drive.") + else: for disk in self._config["drives"]: info ("Ignoring drive type argument: ", disk["type"]) command += self.drive_arg(disk["file"], disk["format"], @@ -255,6 +257,7 @@ def boot(self, multiboot, debug=False, kernel_args = "", image_name = None): command += [kernel_args] try: + self.info("Starting ", command) self.start_process(command) self.info("Started process PID ",self._proc.pid) except Exception as e: From 7c42fc36cead5829e073765ef286b1eced61ff33 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 30 Jul 2018 13:21:04 +0200 Subject: [PATCH 493/723] solo5: Minor solo5blk changes --- src/drivers/solo5blk.cpp | 2 -- src/platform/x86_solo5/os.cpp | 3 --- 2 files changed, 5 deletions(-) diff --git a/src/drivers/solo5blk.cpp b/src/drivers/solo5blk.cpp index b6edf49207..cc6b2fada4 100644 --- a/src/drivers/solo5blk.cpp +++ b/src/drivers/solo5blk.cpp @@ -36,7 +36,6 @@ Solo5Blk::buffer_t Solo5Blk::read_sync(block_t blk) { res = solo5_block_read((solo5_off_t) blk * SECTOR_SIZE, buffer->data(), SECTOR_SIZE); if (res != SOLO5_R_OK) { - buffer.reset(); return nullptr; } return buffer; @@ -51,7 +50,6 @@ Solo5Blk::buffer_t Solo5Blk::read_sync(block_t blk, size_t count) { res = solo5_block_read((solo5_off_t) (blk + i) * SECTOR_SIZE, data + (i * SECTOR_SIZE), SECTOR_SIZE); if (res != SOLO5_R_OK) { - buffer.reset(); return nullptr; } } diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index 0740e1f79d..f6b4761a95 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -213,6 +213,3 @@ void OS::block() // Decrement level blocking_level -= 1; } - -extern "C" -void __os_store_soft_reset(void*, size_t) {} From 5014cc7d0678989fef270d5cf11b953e9effcd36 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 30 Jul 2018 13:52:24 +0200 Subject: [PATCH 494/723] kernel: Add Timers::next() --- api/kernel/timers.hpp | 4 ++++ src/kernel/timers.cpp | 15 +++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/api/kernel/timers.hpp b/api/kernel/timers.hpp index d44f74d6d9..73c0a12cc4 100644 --- a/api/kernel/timers.hpp +++ b/api/kernel/timers.hpp @@ -50,6 +50,10 @@ class Timers /// returns the number of free timers static size_t free(); + /// returns the time to next timer, or zero + /// implementations may treat 1 nanosecond as "timer activation imminent" + static duration_t next(); + /// NOTE: All above operations operate on the current CPU /// NOTE: There is a separate timer system on each active CPU diff --git a/src/kernel/timers.cpp b/src/kernel/timers.cpp index 18297e857a..220f2a97bd 100644 --- a/src/kernel/timers.cpp +++ b/src/kernel/timers.cpp @@ -200,6 +200,21 @@ size_t Timers::free() { /// scheduling /// +duration_t Timers::next() +{ + auto& system = get(); + if (LIKELY(!system.scheduled.empty())) + { + auto it = system.scheduled.begin(); + auto when = it->first; + auto diff = when - now(); + // avoid returning zero or negative diff + if (diff < nanoseconds(1)) return nanoseconds(1); + return diff; + } + return duration_t::zero(); +} + void Timers::timers_handler() { auto& system = get(); From f2358cb68e3b4ef7e683122c1d9c03cfd30defc3 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 30 Jul 2018 13:53:22 +0200 Subject: [PATCH 495/723] solo5: Wait for next timer in event loop --- src/platform/x86_solo5/os.cpp | 33 ++++++++++++++----- .../integration/virtio_block/CMakeLists.txt | 3 +- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index f6b4761a95..0c83b1080b 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -142,21 +142,38 @@ void OS::start(const char* cmdline) void OS::event_loop() { - while (power_) { - int rc; - + while (power_) + { // add a global symbol here so we can quickly discard // event loop from stack sampling asm volatile( ".global _irq_cb_return_location;\n" "_irq_cb_return_location:" ); - // XXX: temporarily ALWAYS sleep for 0.5 ms. We should ideally ask Timers - // for the next immediate timer to fire (the first from the "scheduled" list - // of timers?) - rc = solo5_yield(solo5_clock_monotonic() + 500000ULL); // now + 0.5 ms + int res = 0; + auto nxt = Timers::next(); + if (nxt == std::chrono::nanoseconds(0)) + { + // no next timer, just wait a while + res = solo5_yield(solo5_clock_monotonic() + 500000000ULL); // 500 ms + //printf("Waiting, next is indeterminate...\n"); + } + else if (nxt == std::chrono::nanoseconds(1)) + { + // there is an imminent or activated timer, don't wait + //printf("Not waiting, imminent timer...\n"); + } + else + { + res = solo5_yield(solo5_clock_monotonic() + nxt.count()); + //printf("Waiting %llu nanos\n", nxt.count()); + } + + // handle any activated timers Timers::timers_handler(); - if (rc) { + if (res != 0) + { + // handle any network traffic for(auto& nic : hw::Devices::devices()) { nic->poll(); break; diff --git a/test/fs/integration/virtio_block/CMakeLists.txt b/test/fs/integration/virtio_block/CMakeLists.txt index f9c3dac86f..28289bed15 100644 --- a/test/fs/integration/virtio_block/CMakeLists.txt +++ b/test/fs/integration/virtio_block/CMakeLists.txt @@ -10,11 +10,10 @@ project (test_virtioblk) set(SERVICE_NAME "VirtioBlk test") set(BINARY "virtioblk") -set(MAX_MEM 128) set(SOURCES service.cpp ) -set(DRIVERS virtioblk) +set(DRIVERS virtioblk solo5blk) include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) From 64e04de23216da965d989dbb5a8a4961c7d7baf2 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 30 Jul 2018 15:33:50 +0200 Subject: [PATCH 496/723] solo5: Fix OS::block(), tweaks --- src/platform/x86_solo5/os.cpp | 87 +++++++++++++++-------------------- 1 file changed, 37 insertions(+), 50 deletions(-) diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index 0c83b1080b..3a0b9cbbb1 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -140,6 +140,38 @@ void OS::start(const char* cmdline) Timers::ready(); } +static inline void event_loop_inner() +{ + int res = 0; + auto nxt = Timers::next(); + if (nxt == std::chrono::nanoseconds(0)) + { + // no next timer, just wait a while + res = solo5_yield(solo5_clock_monotonic() + 500000000ULL); // 500 ms + //printf("Waiting, next is indeterminate...\n"); + } + else if (nxt == std::chrono::nanoseconds(1)) + { + // there is an imminent or activated timer, don't wait + //printf("Not waiting, imminent timer...\n"); + } + else + { + res = solo5_yield(solo5_clock_monotonic() + nxt.count()); + //printf("Waiting %llu nanos\n", nxt.count()); + } + + // handle any activated timers + Timers::timers_handler(); + if (res != 0) + { + // handle any network traffic + for(auto& nic : hw::Devices::devices()) { + nic->poll(); + } + } +} + void OS::event_loop() { while (power_) @@ -150,35 +182,7 @@ void OS::event_loop() ".global _irq_cb_return_location;\n" "_irq_cb_return_location:" ); - int res = 0; - auto nxt = Timers::next(); - if (nxt == std::chrono::nanoseconds(0)) - { - // no next timer, just wait a while - res = solo5_yield(solo5_clock_monotonic() + 500000000ULL); // 500 ms - //printf("Waiting, next is indeterminate...\n"); - } - else if (nxt == std::chrono::nanoseconds(1)) - { - // there is an imminent or activated timer, don't wait - //printf("Not waiting, imminent timer...\n"); - } - else - { - res = solo5_yield(solo5_clock_monotonic() + nxt.count()); - //printf("Waiting %llu nanos\n", nxt.count()); - } - - // handle any activated timers - Timers::timers_handler(); - if (res != 0) - { - // handle any network traffic - for(auto& nic : hw::Devices::devices()) { - nic->poll(); - break; - } - } + event_loop_inner(); } @@ -201,31 +205,14 @@ void OS::halt() { os_cycles_hlt += solo5_clock_monotonic() - cycles_before; } -// Keep track of blocking levels -static uint32_t blocking_level = 0; -static uint32_t highest_blocking_level = 0; - void OS::block() { - // Increment level + static uint32_t blocking_level = 0; blocking_level += 1; + // prevent recursion stack overflow + assert(blocking_level < 200); - // Increment highest if applicable - if (blocking_level > highest_blocking_level) - highest_blocking_level = blocking_level; - - int rc; - rc = solo5_yield(solo5_clock_monotonic() + 50000ULL); // now + 0.05 ms - if (rc == 0) { - Timers::timers_handler(); - } else { - - for(auto& nic : hw::Devices::devices()) { - nic->poll(); - break; - } - - } + event_loop_inner(); // Decrement level blocking_level -= 1; From 848abaec3f6b04972baa39671ad7217c346bb0f7 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Mon, 30 Jul 2018 16:51:17 +0200 Subject: [PATCH 497/723] buddy_alloc: start pool as low as possible --- api/util/alloc_buddy.hpp | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index de5f71abab..ddef49b544 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -187,9 +187,7 @@ namespace mem::buddy { using namespace util; Expects(bits::is_pow2(pool_size_)); Expects(pool_size_ >= min_size); - Expects(bits::is_aligned(start)); Expects(bits::is_aligned(start_addr_)); - Ensures(pool_size_ + nodes_.size() * sizeof(Node_t) <= bufsize); // Initialize nodes memset(nodes_.data(), 0, nodes_.size() * sizeof(Node_arr::element_type)); @@ -198,11 +196,11 @@ namespace mem::buddy { static Alloc* create(void* addr, Size_t bufsize) { using namespace util; Size_t pool_size_ = pool_size(bufsize); - Size_t overhead_ = bufsize - pool_size_; - auto* nodes_ptr = (char*)addr + overhead_; Expects(bufsize >= required_size(pool_size_)); - Ensures(bufsize >= required_size(pool_size_)); - auto* alloc = new (addr) Alloc(nodes_ptr, bufsize, pool_size_); + + // Placement new an allocator on addr, passing in the rest of memory + auto* alloc_begin = (char*)addr + sizeof(Alloc); + auto* alloc = new (addr) Alloc(alloc_begin, bufsize, pool_size_); return alloc; } @@ -572,21 +570,21 @@ namespace mem::buddy { std::string dashes(80, '-'); out << dashes << "\n"; out << "Bytes used: " << util::Byte_r(bytes_used()) - << " Bytes free: " << util::Byte_r(bytes_free()) - << " H: " << std::dec << tree_height() - << " W: " << tree_width() - << " Alloc.size: " << util::Byte_r(sizeof(*this)) << "\n" - << "Address pool: 0x" << std::hex - << root().addr() << " - " << root().addr() + pool_size_ - << std::dec << "\n"; + << " Bytes free: " << util::Byte_r(bytes_free()) + << " H: " << std::dec << tree_height() + << " W: " << tree_width() + << " Alloc.size: " << util::Byte_r(sizeof(*this)) << "\n" + << "Address pool: 0x" << std::hex + << root().addr() << " - " << root().addr() + pool_size_ + << std::dec << " ( " << util::Byte_r(pool_size()) <<" ) \n"; auto track = alloc_tracker(); out << "Allocations: " << track.allocs - << " Steps: Last: " << track.last - << " Min: " << track.min - << " Max: " << track.max - << " Total: " << track.total - << " Avg: " << track.total / track.allocs - << "\n"; + << " Steps: Last: " << track.last + << " Min: " << track.min + << " Max: " << track.max + << " Total: " << track.total + << " Avg: " << track.total / track.allocs + << "\n"; out << dashes << "\n"; return out.str(); } From 1cb689d8410693961c4bec9eeec6726f02cd3a06 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 31 Jul 2018 11:01:28 +0200 Subject: [PATCH 498/723] cmake: Add __includeos__ global define --- CMakeLists.txt | 2 +- cmake/post.service.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d15a65969a..ac21b6a095 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -62,7 +62,7 @@ set(CAPABS "${CAPABS} -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_P # Various global defines # * NO_DEBUG disables output from the debug macro # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures -set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE") +set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE -D__includeos__") set(WARNS "-Wall -Wextra") # -Werror # configure options diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index f70d239354..ecbdac5bca 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -43,7 +43,7 @@ endif() # Various global defines # * OS_TERMINATE_ON_CONTRACT_VIOLATION provides classic assert-like output from Expects / Ensures # * _GNU_SOURCE enables POSIX-extensions in newlib, such as strnlen. ("everything newlib has", ref. cdefs.h) -set(CAPABS "${CAPABS} -fstack-protector-strong -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE -DSERVICE=\"\\\"${BINARY}\\\"\" -DSERVICE_NAME=\"\\\"${SERVICE_NAME}\\\"\"") +set(CAPABS "${CAPABS} -fstack-protector-strong -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCPP_HAS_MUSL_LIBC -D_GNU_SOURCE -D__includeos__ -DSERVICE=\"\\\"${BINARY}\\\"\" -DSERVICE_NAME=\"\\\"${SERVICE_NAME}\\\"\"") set(WARNS "-Wall -Wextra") #-pedantic # Compiler optimization From 37b8472da30c9e32888723fcaa5d0a75bbc52778 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 31 Jul 2018 12:57:43 +0200 Subject: [PATCH 499/723] solo5: Wait longer, process events --- src/platform/x86_solo5/os.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index 3a0b9cbbb1..fe1a284cac 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -137,7 +138,7 @@ void OS::start(const char* cmdline) // timer stop function [] () {}); - Timers::ready(); + Events::get().defer(Timers::ready); } static inline void event_loop_inner() @@ -146,9 +147,11 @@ static inline void event_loop_inner() auto nxt = Timers::next(); if (nxt == std::chrono::nanoseconds(0)) { - // no next timer, just wait a while - res = solo5_yield(solo5_clock_monotonic() + 500000000ULL); // 500 ms - //printf("Waiting, next is indeterminate...\n"); + // no next timer, wait forever + //printf("Waiting 15s, next is indeterminate...\n"); + const unsigned long long count = 15000000000ULL; + res = solo5_yield(solo5_clock_monotonic() + count); + os_cycles_hlt += count; } else if (nxt == std::chrono::nanoseconds(1)) { @@ -157,12 +160,14 @@ static inline void event_loop_inner() } else { - res = solo5_yield(solo5_clock_monotonic() + nxt.count()); //printf("Waiting %llu nanos\n", nxt.count()); + res = solo5_yield(solo5_clock_monotonic() + nxt.count()); + os_cycles_hlt += nxt.count(); } // handle any activated timers Timers::timers_handler(); + Events::get().process_events(); if (res != 0) { // handle any network traffic From c299458791ad991187bee58c8720fed59d2ac7e4 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 31 Jul 2018 12:57:58 +0200 Subject: [PATCH 500/723] examples: Make TCP_perf work on solo5 --- examples/TCP_perf/CMakeLists.txt | 4 +++- examples/TCP_perf/service.cpp | 22 ++++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/examples/TCP_perf/CMakeLists.txt b/examples/TCP_perf/CMakeLists.txt index 501e5917df..9b3c165586 100644 --- a/examples/TCP_perf/CMakeLists.txt +++ b/examples/TCP_perf/CMakeLists.txt @@ -20,13 +20,15 @@ set(SOURCES if ("$ENV{PLATFORM}" STREQUAL "x86_solo5") set(DRIVERS - solo5net # Virtio networking + solo5net + #solo5blk ) else() set(DRIVERS virtionet vmxnet3 e1000 + boot_logger ) endif() diff --git a/examples/TCP_perf/service.cpp b/examples/TCP_perf/service.cpp index 18e3dec92f..bf9ae25eb6 100644 --- a/examples/TCP_perf/service.cpp +++ b/examples/TCP_perf/service.cpp @@ -16,16 +16,21 @@ // limitations under the License. #include +#include #include #include #include #include -#define ENABLE_JUMBO_FRAMES using namespace net::tcp; size_t bufsize = 128*1024; -uint32_t SIZE = 1024*1024*512; +#ifdef PLATFORM_x86_solo5 +static const uint32_t SIZE = 1024*1024*50; +#else +static const uint32_t SIZE = 1024*1024*512; +#define ENABLE_JUMBO_FRAMES +#endif uint64_t packets_rx{0}; uint64_t packets_tx{0}; uint64_t received{0}; @@ -38,8 +43,13 @@ bool SACK{true}; struct activity { void reset() { +#ifdef PLATFORM_x86_solo5 + total = 0; + asleep = 0; +#else total = StackSampler::samples_total(); asleep = StackSampler::samples_asleep(); +#endif } void print(activity& other) { auto tdiff = total - other.total; @@ -70,16 +80,18 @@ void start_measure() dack.count(), winsize, wscale, (winsize << wscale)/1024, timestamps ? "ON" : "OFF", SACK ? "ON" : "OFF"); - ts = OS::nanos_since_boot(); + ts = RTC::nanos_now(); activity_before.reset(); } void stop_measure() { - auto diff = OS::nanos_since_boot() - ts; + auto diff = RTC::nanos_now() - ts; activity_after.reset(); +#ifndef PLATFORM_x86_solo5 StackSampler::print(15); +#endif activity_after.print(activity_before); packets_rx = Statman::get().get_by_name("eth0.ethernet.packets_rx").get_uint64() - packets_rx; @@ -95,8 +107,10 @@ void Service::start() {} void Service::ready() { +#ifndef PLATFORM_x86_solo5 StackSampler::begin(); StackSampler::set_mode(StackSampler::MODE_DUMMY); +#endif static auto blob = net::tcp::construct_buffer(SIZE); From b312397db11d8d130b05ff7ee658f37619c1770b Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 31 Jul 2018 13:36:39 +0200 Subject: [PATCH 501/723] examples: Move out old service in LiveUpdate example --- examples/LiveUpdate/{ => tls}/drive/test.key | 0 examples/LiveUpdate/{ => tls}/drive/test.pem | 0 examples/LiveUpdate/{ => tls}/service.cpp | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename examples/LiveUpdate/{ => tls}/drive/test.key (100%) rename examples/LiveUpdate/{ => tls}/drive/test.pem (100%) rename examples/LiveUpdate/{ => tls}/service.cpp (100%) diff --git a/examples/LiveUpdate/drive/test.key b/examples/LiveUpdate/tls/drive/test.key similarity index 100% rename from examples/LiveUpdate/drive/test.key rename to examples/LiveUpdate/tls/drive/test.key diff --git a/examples/LiveUpdate/drive/test.pem b/examples/LiveUpdate/tls/drive/test.pem similarity index 100% rename from examples/LiveUpdate/drive/test.pem rename to examples/LiveUpdate/tls/drive/test.pem diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/tls/service.cpp similarity index 100% rename from examples/LiveUpdate/service.cpp rename to examples/LiveUpdate/tls/service.cpp From 13bc536e540a951aa26849ebb81918bd3fe8aeca Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 31 Jul 2018 14:31:00 +0200 Subject: [PATCH 502/723] examples: LiveUpdate now uses IRC server instead --- examples/IRCd/CMakeLists.txt | 3 +- examples/LiveUpdate/CMakeLists.txt | 19 ++++ examples/LiveUpdate/config.json | 19 ++++ examples/LiveUpdate/service.cpp | 148 +++++++++++++++++++++++++++++ 4 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 examples/LiveUpdate/service.cpp diff --git a/examples/IRCd/CMakeLists.txt b/examples/IRCd/CMakeLists.txt index 177e4f80f1..7281611bc4 100644 --- a/examples/IRCd/CMakeLists.txt +++ b/examples/IRCd/CMakeLists.txt @@ -41,8 +41,9 @@ option(REAL_HW "Run on real hardware" ON) if (REAL_HW) set(DRIVERS - #boot_logger + boot_logger e1000 + vmxnet3 vga_output vga_emergency ) diff --git a/examples/LiveUpdate/CMakeLists.txt b/examples/LiveUpdate/CMakeLists.txt index b0f68dd86d..b17ce73d9b 100644 --- a/examples/LiveUpdate/CMakeLists.txt +++ b/examples/LiveUpdate/CMakeLists.txt @@ -16,11 +16,30 @@ set(BINARY "liveupdate") # Source files to be linked with OS library parts to form bootable image set(SOURCES service.cpp + ../IRCd/autoconf.cpp + ../IRCd/ircd/channel.cpp + ../IRCd/ircd/client.cpp + ../IRCd/ircd/client_commands.cpp + ../IRCd/ircd/client_new.cpp + ../IRCd/ircd/client_send.cpp + ../IRCd/ircd/client_timeout.cpp + ../IRCd/ircd/ircd.cpp + ../IRCd/ircd/ircsplit.cpp + ../IRCd/ircd/modes.cpp + ../IRCd/ircd/readq.cpp + ../IRCd/ircd/restore.cpp + ../IRCd/ircd/selftest.cpp + ../IRCd/ircd/server.cpp + ../IRCd/ircd/server_commands.cpp + ../IRCd/ircd/test.cpp + ../IRCd/ircd/transform.cpp ) # DRIVERS / PLUGINS: set(DRIVERS virtionet + vmxnet3 + #boot_logger ) set(PLUGINS diff --git a/examples/LiveUpdate/config.json b/examples/LiveUpdate/config.json index 7d29cc16da..b3bf8c52bd 100644 --- a/examples/LiveUpdate/config.json +++ b/examples/LiveUpdate/config.json @@ -8,8 +8,27 @@ "gateway": "10.0.0.1" } ], + "terminal": { "iface": 0, "port": 23 + }, + + "uplink": { + "iface": 0, + "url": "10.20.17.12:9090", + "token": "kappa123", + "reboot": true + }, + + "ircd": { + "client_iface": 0, + "client_port": 6667, + "server_iface": 0, + "server_port": 7000, + "server_id": 0, + "server_name": "irc.includeos.org", + "server_netw": "IncludeNet" } + } diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/service.cpp new file mode 100644 index 0000000000..3b04128483 --- /dev/null +++ b/examples/LiveUpdate/service.cpp @@ -0,0 +1,148 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#define USE_STACK_SAMPLING +#define PERIOD_SECS 4 + +#include "../IRCd/ircd/ircd.hpp" +static std::unique_ptr ircd = nullptr; +using namespace std::chrono; + +void Service::start() +{ + // run a small self-test to verify parser is sane + extern void selftest(); selftest(); + + ircd = IrcServer::from_config(); + + ircd->set_motd([] () -> const std::string& { + static const std::string motd = R"M0TDT3XT( + .-') _ _ .-') _ ('-. .-') + ( OO ) ) ( ( OO) ) _( OO) ( OO ). + ,-.-') ,--./ ,--,' .-----. ,--. ,--. ,--. \ .'_ (,------. .-'),-----. (_)---\_) + | |OO)| \ | |\ ' .--./ | |.-') | | | | ,`'--..._) | .---' ( OO' .-. '/ _ | + | | \| \| | )| |('-. | | OO )| | | .-') | | \ ' | | / | | | |\ :` `. + | |(_/| . |//_) |OO )| |`-' || |_|( OO )| | ' |(| '--. \_) | |\| | '..`''.) + ,| |_.'| |\ | || |`-'|(| '---.'| | | `-' /| | / : | .--' \ | | | |.-._) \ +(_| | | | \ |(_' '--'\ | |(' '-'(_.-' | '--' / | `---. `' '-' '\ / + `--' `--' `--' `-----' `------' `-----' `-------' `------' `-----' `-----' +)M0TDT3XT"; + return motd; + }); + // motd spam + //printf("%s\n", ircd->get_motd().c_str()); +} + +#include +std::string now() +{ + auto tnow = time(0); + auto* curtime = localtime(&tnow); + + char buff[48]; + int len = strftime(buff, sizeof(buff), "%c", curtime); + return std::string(buff, len); +} + +void print_heap_info() +{ + static intptr_t last = 0; + // show information on heap status, to discover leaks etc. + auto heap_begin = OS::heap_begin(); + auto heap_end = OS::heap_end(); + auto heap_usage = OS::heap_usage(); + intptr_t heap_size = heap_end - heap_begin; + last = heap_size - last; + printf("Heap begin %#lx size %lu Kb\n", heap_begin, heap_size / 1024); + printf("Heap end %#lx diff %lu (%ld Kb)\n", heap_end, last, last / 1024); + printf("Heap usage %lu kB\n", heap_usage / 1024); + last = (int32_t) heap_size; +} + +#include +void print_stats(int) +{ +#ifdef USE_STACK_SAMPLING + StackSampler::set_mask(true); +#endif + + static std::deque M; + static int last = 0; + // only keep 5 measurements + if (M.size() > 4) M.pop_front(); + + int diff = ircd->get_counter(STAT_TOTAL_CONNS) - last; + last = ircd->get_counter(STAT_TOTAL_CONNS); + // @PERIOD_SECS between measurements + M.push_back(diff / PERIOD_SECS); + + double cps = 0.0; + for (int C : M) cps += C; + cps /= M.size(); + + printf("[%s] Conns/sec %.1f Heap %.1f kb\n", + now().c_str(), cps, OS::heap_usage() / 1024.0); + // client and channel stats + auto& inet = net::Inet::stack<0>(); + + printf("Syns: %u Conns: %lu Users: %u RAM: %lu bytes Chans: %u\n", + ircd->get_counter(STAT_TOTAL_CONNS), + inet.tcp().active_connections(), + ircd->get_counter(STAT_LOCAL_USERS), + ircd->club(), + ircd->get_counter(STAT_CHANNELS)); + printf("*** ---------------------- ***\n"); +#ifdef USE_STACK_SAMPLING + // stack sampler results + StackSampler::print(10); + printf("*** ---------------------- ***\n"); +#endif + // heap statistics + print_heap_info(); + printf("*** ---------------------- ***\n"); + +#ifdef USE_STACK_SAMPLING + StackSampler::set_mask(false); +#endif +} + +#include "liu.hpp" +static void save_state(liu::Storage& store, const liu::buffer_t*) +{ + +} + +void Service::ready() +{ +#ifdef USE_STACK_SAMPLING + StackSampler::begin(); + //StackSampler::set_mode(StackSampler::MODE_CALLER); +#endif + + Timers::periodic(seconds(5), seconds(PERIOD_SECS), print_stats); + + // raw TCP liveupdate server + auto& inet = net::Super_stack::get(0); + setup_liveupdate_server(inet, 666, save_state); + + // profiler statistics + printf("%s\n", ScopedProfiler::get_statistics(false).c_str()); +} From a8909cd4f7d2a67c3c1657747878430890036864 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 31 Jul 2018 14:32:06 +0200 Subject: [PATCH 503/723] ircd: Add lnotice command, LNOTICE when live updated --- examples/IRCd/ircd/client_new.cpp | 4 ++-- examples/IRCd/ircd/ircd.cpp | 19 +++++++++++++++++-- examples/IRCd/ircd/ircd.hpp | 3 ++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/examples/IRCd/ircd/client_new.cpp b/examples/IRCd/ircd/client_new.cpp index 26c5af2079..7ce3f3123e 100644 --- a/examples/IRCd/ircd/client_new.cpp +++ b/examples/IRCd/ircd/client_new.cpp @@ -13,7 +13,7 @@ void Client::handle_new(const std::vector& msg) { volatile ScopedProfiler profile; const std::string& cmd = msg[0]; - + if (cmd == TK_CAP) { // ignored completely @@ -68,7 +68,7 @@ void Client::auth_notice() send_raw(auth_host, sizeof auth_host - 1); send_raw(auth_idnt, sizeof auth_idnt - 1); //hostname_lookup() - this->host_ = conn->remote().address().str(); + this->host_ = conn->remote().address().to_string(); //ident_check() } void Client::welcome(uint8_t newreg) diff --git a/examples/IRCd/ircd/ircd.cpp b/examples/IRCd/ircd/ircd.cpp index fa8b49037b..4d714d0210 100644 --- a/examples/IRCd/ircd/ircd.cpp +++ b/examples/IRCd/ircd/ircd.cpp @@ -75,7 +75,8 @@ IrcServer::IrcServer( server_stack().ifname().c_str(), sv_port); /// LiveUpdate /// - if (this->init_liveupdate() == false) + const bool live_updated = this->init_liveupdate(); + if (live_updated == false) { // set timestamp for when the server was started this->created_ts = create_timestamp(); @@ -87,6 +88,9 @@ IrcServer::IrcServer( INFO2("Server started on %s", created_string.c_str()); INFO2("Version " IRC_SERVER_VERSION); INFO("IRC", "Server open"); + if (live_updated) { + this->lnotice(this->name(), "Server was just live updated!"); + } } void IrcServer::new_registered_client() @@ -123,6 +127,17 @@ void IrcServer::free_channel(Channel& ch) dec_counter(STAT_CHANNELS); } +void IrcServer::lnotice(const std::string& src, const std::string& msg) +{ + for (size_t id = 0; id < clients.size(); id++) + { + auto& cl = clients.get(id); + if (cl.is_reg()) { + cl.send_from(src, "NOTICE " + cl.nickuserhost() + " :" + msg); + } + } +} + void IrcServer::user_bcast(clindex_t idx, const std::string& from, uint16_t tk, const std::string& msg) { char buffer[256]; @@ -260,7 +275,7 @@ void IrcServer::begin_netburst(Server& target) } } /// clients - for (size_t id = 0; id < channels.size(); id++) + for (size_t id = 0; id < clients.size(); id++) { auto& cl = clients.get(id); if (cl.is_reg()) { diff --git a/examples/IRCd/ircd/ircd.hpp b/examples/IRCd/ircd/ircd.hpp index ec7b45577f..f412a54cae 100644 --- a/examples/IRCd/ircd/ircd.hpp +++ b/examples/IRCd/ircd/ircd.hpp @@ -109,7 +109,8 @@ class IrcServer { // message local operators void wallops(const std::string&); // propagate message globally - void broadcast(net::tcp::buffer_t, size_t); + void lnotice(const std::string& src, const std::string&); + void gnotice(const std::string& src, const std::string&); // send message to all users visible to user, including user void user_bcast(clindex_t user, const char* buffer, size_t len); void user_bcast(clindex_t user, const std::string& from, uint16_t tk, const std::string&); From e81e5c174ed3fbfe172a3ee703aabec0232d8bbf Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 31 Jul 2018 15:40:39 +0200 Subject: [PATCH 504/723] buddy alloc: make const noexcept where possible --- api/util/alloc_buddy.hpp | 129 ++++++++++-------- .../unit/{buddy.cpp => buddy_alloc_test.cpp} | 0 2 files changed, 71 insertions(+), 58 deletions(-) rename test/util/unit/{buddy.cpp => buddy_alloc_test.cpp} (100%) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index ddef49b544..27bdc3051c 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -80,7 +80,6 @@ namespace mem::buddy { } - /** Total allocatable memory in an allocator created in a bufsize buffer **/ static Size_t pool_size(Size_t bufsize){ using namespace util; @@ -125,32 +124,32 @@ namespace mem::buddy { return node_count() / 2 + 1; } - int max_chunksize() const noexcept { + Size_t max_chunksize() const noexcept { // Same as pool size. return min_size << (tree_height() - 1); } - Size_t pool_size() - { return pool_size_; } - + Size_t pool_size() const noexcept { + return pool_size_; + } - Size_t bytes_used() { + Size_t bytes_used() const noexcept { return bytes_used_; } - Addr_t highest_used() { - return root().highest_used(); + Addr_t highest_used() const noexcept { + return root().highest_used_r(); } - Size_t bytes_free(){ + Size_t bytes_free() const noexcept { return pool_size_ - bytes_used_; } - Size_t bytes_free_r(){ - return pool_size_ - root().bytes_used(); + Size_t bytes_free_r() const noexcept { + return pool_size_ - root().bytes_used_r(); } - Size_t bytes_used_r() { + Size_t bytes_used_r() const noexcept { return root().bytes_used_r(); } @@ -162,15 +161,15 @@ namespace mem::buddy { return root().is_free(); } - Addr_t addr_begin() { + Addr_t addr_begin() const noexcept { return reinterpret_cast(start_addr_); } - Addr_t addr_end() { + Addr_t addr_end() const noexcept { return start_addr_ + pool_size_; } - bool in_range(void* a) { + bool in_range(void* a) const noexcept { auto addr = reinterpret_cast(a); return addr >= addr_begin() and addr < addr_end(); } @@ -205,7 +204,7 @@ namespace mem::buddy { } - Size_t chunksize(Size_t wanted_sz) { + Size_t chunksize(Size_t wanted_sz) const noexcept { auto sz = util::bits::next_pow2(wanted_sz); if (sz > max_chunksize()) return 0; @@ -226,7 +225,7 @@ namespace mem::buddy { int allocs; }; - Track_res alloc_tracker(Track action = Track::get) { + Track_res alloc_tracker(Track action = Track::get) const noexcept { if constexpr (Track_allocs) { static Track_res tr {0,0,0,0,0}; if (action == Track::start) { @@ -250,12 +249,11 @@ namespace mem::buddy { return {}; } - void* allocate(Size_t size) { + void* allocate(Size_t size) noexcept { Expects(start_addr_); auto node = root(); - auto addr = start_addr_; auto sz = chunksize(size); if (not sz) return 0; @@ -280,101 +278,108 @@ namespace mem::buddy { } + //private: class Node_view { public: - Node_view(int index, Alloc* alloc) + Node_view(int index, const Alloc* alloc) : i_{index}, alloc_{alloc}, my_size_{compute_size()}, my_addr_{compute_addr()} {} Node_view() = default; - Node_view left() { + Node_view empty() const noexcept { + return Node_view(); + } + + Node_view left() const noexcept { if (is_leaf()) { - return Node_view(); + return empty(); } return Node_view(i_ * 2 + 1, alloc_); } - Node_view right() { + Node_view right() const noexcept { if (is_leaf()) { - return Node_view(); + return empty(); } return Node_view(i_ * 2 + 2, alloc_); } - Addr_t addr() - { return my_addr_; } + Addr_t addr() const noexcept { + return my_addr_; + } - Size_t size() - { return my_size_; } + Size_t size() const noexcept { + return my_size_; + } - bool is_leaf() { + bool is_leaf() const noexcept { return i_ >= alloc_->nodes_.size() / 2; } - bool is_taken() { + bool is_taken() const noexcept { return data() & Flags::taken; } - bool is_free() { + bool is_free() const noexcept { return data() == 0; } - bool is_full() { + bool is_full() const noexcept { auto fl = data(); return fl & Flags::taken or (fl & Flags::left_full and fl & Flags::right_full); } - bool in_range(Addr_t a) { + bool in_range(Addr_t a) const noexcept { return a >= my_addr_ and a < my_addr_ + my_size_; } - void set_flag(Flags f){ + void set_flag(Flags f) { data() |= f; } - void clear_flag(Flags f){ + void clear_flag(Flags f) { data() &= ~f; } - bool is_parent() { + bool is_parent() const noexcept { return i_ <= alloc_->leaf0(); } - bool is_full_r() { + bool is_full_r() const noexcept { if (is_taken()) return true; return is_parent() and left().is_full_r() and is_parent() and right().is_full_r(); } - bool is_free_r() { + bool is_free_r() const noexcept { auto lfree = not is_parent() or left().is_free_r(); auto rfree = not is_parent() or right().is_free_r(); return not is_taken() and lfree and rfree; } - int height() { + int height() const noexcept { return (util::bits::fls(i_ + 1)); } - Size_t compute_size() { + Size_t compute_size() const noexcept { return min_size << (alloc_->tree_height() - height()); } - Addr_t compute_addr() { + Addr_t compute_addr() const noexcept { auto first_lvl_idx = 1 << (height() - 1); auto sz_offs = i_ - first_lvl_idx + 1; return alloc_->start_addr_ + (sz_offs * my_size_); } - operator bool() { + operator bool() const noexcept { return alloc_->start_addr_ != 0 and alloc_ != nullptr; } @@ -382,6 +387,10 @@ namespace mem::buddy { return alloc_->nodes_.at(i_); } + const Node_t& data() const noexcept { + return alloc_->nodes_.at(i_); + } + Addr_t allocate_self() { Expects(not (data() & Flags::taken)); set_flag(Flags::taken); @@ -473,16 +482,16 @@ namespace mem::buddy { return res; } - Size_t dealloc_self(){ + Size_t dealloc_self() { data() = 0; return my_size_; } - bool is_left(Addr_t ptr){ + bool is_left(Addr_t ptr) const noexcept { return ptr < my_addr_ + my_size_ / 2; } - bool is_right(Addr_t ptr) { + bool is_right(Addr_t ptr) const noexcept { return ptr >= my_addr_ + my_size_ / 2; } @@ -527,45 +536,49 @@ namespace mem::buddy { return 0; } - Size_t bytes_used() { + Size_t bytes_used_r() const noexcept { if (is_taken()) return my_size_; if (is_parent()) - return left().bytes_used() + right().bytes_used(); + return left().bytes_used_r() + right().bytes_used_r(); return 0; } - Addr_t highest_used() { + Addr_t highest_used_r() const noexcept { if (is_taken() or not is_parent()) return my_addr_ + my_size_; - auto rhs = right().highest_used(); + auto rhs = right().highest_used_r(); if (rhs > my_addr_ + my_size_) return rhs; - return left().highest_used(); + return left().highest_used_r(); } - std::string to_string(){ + std::string to_string() const { std::stringstream out; out << std::hex - // << addr()<< " | " - << (int)data() //is_full_r() // + << (int)data() << std::dec; return out.str(); } private: int i_ = 0; - Alloc* alloc_ = nullptr; + const Alloc* alloc_ = nullptr; Size_t my_size_ = 0; Addr_t my_addr_ = 0; }; - Node_view root() - { return Node_view(0, this); } + Node_view root() { + return Node_view(0, this); + } + + const Node_view root() const { + return Node_view(0, this); + } - std::string summary() { + std::string summary() const { std::stringstream out; std::string dashes(80, '-'); out << dashes << "\n"; @@ -588,7 +601,7 @@ namespace mem::buddy { out << dashes << "\n"; return out.str(); } - std::string draw_tree(bool index = false) { + std::string draw_tree(bool index = false) const { auto node_pr_w = index ? 6 : 4; auto line_w = tree_width() * node_pr_w; diff --git a/test/util/unit/buddy.cpp b/test/util/unit/buddy_alloc_test.cpp similarity index 100% rename from test/util/unit/buddy.cpp rename to test/util/unit/buddy_alloc_test.cpp From 1de7bb627b599f500c663ccea8989e27b0d58038 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 31 Jul 2018 16:00:46 +0200 Subject: [PATCH 505/723] test: rename buddy allocator test --- test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cab4944065..4fe9f88e56 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -158,7 +158,7 @@ set(TEST_SOURCES ${TEST}/util/unit/uri_test.cpp ${TEST}/util/unit/bitops.cpp ${TEST}/util/unit/lstack.cpp - ${TEST}/util/unit/buddy.cpp + ${TEST}/util/unit/buddy_alloc_test.cpp ) set(OS_SOURCES From 3cd3cab685fa20324ccee1819ab08a696a4015e9 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 31 Jul 2018 22:45:08 +0200 Subject: [PATCH 506/723] buddy alloc: add std::allocator and pmr::memory_resource interfaces --- api/util/alloc_buddy.hpp | 66 ++++++++++++++++++--- test/util/unit/buddy_alloc_test.cpp | 92 ++++++++++++++++++++++++++++- 2 files changed, 150 insertions(+), 8 deletions(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index 27bdc3051c..969fcbf9d6 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -20,12 +20,19 @@ #include #include #include -#include -#include -#include +#if __has_include() +#include +#else +#include +namespace std::pmr { + using memory_resource = std::experimental::pmr::memory_resource; +} +#endif #include #include +#include +#include #include // @@ -71,7 +78,7 @@ namespace mem::buddy { * A buddy allocator over a fixed size pool **/ template - struct Alloc { + struct Alloc : public std::pmr::memory_resource { static constexpr mem::buddy::Size_t min_size = 4096; static constexpr mem::buddy::Size_t align = min_size; @@ -273,12 +280,26 @@ namespace mem::buddy { bytes_used_ -= res; } + void* do_allocate(std::size_t bytes, std::size_t alignment) override { + using namespace util; + auto aligned_size = bits::roundto(alignment, bytes); + return allocate(aligned_size); + } + + void do_deallocate(void* p, std::size_t bytes, std::size_t alignment) override { + using namespace util; + deallocate(p, bits::roundto(alignment, bytes)); + } + + bool do_is_equal(const memory_resource& other) const noexcept override { + return &other == this; + } + void free(void* addr) { deallocate(addr, 0); } - //private: class Node_view { public: @@ -583,10 +604,10 @@ namespace mem::buddy { std::string dashes(80, '-'); out << dashes << "\n"; out << "Bytes used: " << util::Byte_r(bytes_used()) - << " Bytes free: " << util::Byte_r(bytes_free()) + << ", Bytes free: " << util::Byte_r(bytes_free()) << " H: " << std::dec << tree_height() << " W: " << tree_width() - << " Alloc.size: " << util::Byte_r(sizeof(*this)) << "\n" + << " Tree size: " << util::Byte_r(nodes_.size()) << "\n" << "Address pool: 0x" << std::hex << root().addr() << " - " << root().addr() + pool_size_ << std::dec << " ( " << util::Byte_r(pool_size()) <<" ) \n"; @@ -639,6 +660,37 @@ namespace mem::buddy { Size_t bytes_used_ = 0; }; + /** + * C++17 std::allocator interface using buddy allocator + **/ + template + struct Allocator { + using value_type = T; + + Allocator(Resource* alloc) + : resource{alloc} + {} + + T* allocate(std::size_t size) { + return reinterpret_cast(resource->allocate(size * sizeof(T))); + } + + void deallocate(T* ptr, std::size_t size) { + resource->deallocate(ptr, size * sizeof(T)); + } + + bool operator==(const Allocator& other) { + return resource == other.resource; + } + + bool operator!=(const Allocator& other) { + return not other == *this; + } + + Resource* resource; + + }; + } #endif diff --git a/test/util/unit/buddy_alloc_test.cpp b/test/util/unit/buddy_alloc_test.cpp index 2207f900c1..c5b7d70a3a 100644 --- a/test/util/unit/buddy_alloc_test.cpp +++ b/test/util/unit/buddy_alloc_test.cpp @@ -17,9 +17,12 @@ // #define DEBUG_UNIT #include -#include #include +#if __has_include() +#include // For pmr::vector +#endif + struct Pool { using Alloc = mem::buddy::Alloc; @@ -297,3 +300,90 @@ CASE("mem::buddy random chaos with data verification"){ EXPECT(alloc.empty()); } + + +CASE("mem::buddy as pmr::memory_resource") { + using namespace util; + using namespace std::experimental; + + Pool pool(1_GiB); + auto* resource = pool.alloc; + + pmr::polymorphic_allocator alloc(resource); + pmr::vector numbers(alloc); + + EXPECT(resource->empty()); + numbers.push_back(10); + EXPECT(not resource->empty()); + EXPECT(resource->bytes_used() == Pool::Alloc::min_size); + numbers.push_back(20); + numbers.push_back(30); + numbers.push_back(40); + EXPECT(resource->bytes_used() == Pool::Alloc::min_size); + + // Force the vector to return memory + numbers.clear(); + numbers.shrink_to_fit(); + EXPECT(resource->empty()); + + // Make sure it still works as expected + numbers.push_back(20); + numbers.push_back(30); + numbers.push_back(40); + EXPECT(resource->bytes_used() == Pool::Alloc::min_size); + + EXPECT((numbers == pmr::vector{20,30,40})); + + numbers.clear(); + numbers.shrink_to_fit(); + EXPECT(resource->empty()); + + #ifdef DEBUG_UNIT + std::cout << resource->summary() << std::endl; + #endif +} + + +CASE("mem::buddy as std::allocator") { + using namespace util; + + Pool pool(1_GiB); + auto* resource = pool.alloc; + + std::vector> numbers(resource); + + EXPECT(resource->empty()); + numbers.push_back(10); + EXPECT(not resource->empty()); + EXPECT(resource->bytes_used() == Pool::Alloc::min_size); + numbers.push_back(20); + numbers.push_back(30); + numbers.push_back(40); + EXPECT(resource->bytes_used() == Pool::Alloc::min_size); + + // Force the vector to return memory + numbers.clear(); + numbers.shrink_to_fit(); + EXPECT(resource->empty()); + + // Make sure it still works as expected + numbers.push_back(20); + numbers.push_back(30); + numbers.push_back(40); + EXPECT(resource->bytes_used() == Pool::Alloc::min_size); + + // Can't compare this vector with any generic vector type + // since the allocator become a part of the vector type + EXPECT(numbers[0] == 20); + EXPECT(numbers[1] == 30); + EXPECT(numbers[2] == 40); + + + numbers.clear(); + numbers.shrink_to_fit(); + EXPECT(resource->empty()); + + #ifdef DEBUG_UNIT + std::cout << resource->summary() << std::endl; + #endif +} From ab37b7920663e465bdd7e52969c37499140a5b66 Mon Sep 17 00:00:00 2001 From: Ricardo Koller Date: Thu, 2 Aug 2018 09:27:17 -0400 Subject: [PATCH 507/723] Point to solo5 285b80a --- cmake/cross_compiled_libraries.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cross_compiled_libraries.cmake b/cmake/cross_compiled_libraries.cmake index 9f9da88b55..c947e180be 100644 --- a/cmake/cross_compiled_libraries.cmake +++ b/cmake/cross_compiled_libraries.cmake @@ -33,7 +33,7 @@ ExternalProject_Add(solo5_repo PREFIX precompiled BUILD_IN_SOURCE 1 GIT_REPOSITORY https://github.com/solo5/solo5.git - GIT_TAG v0.3.0 + GIT_TAG 285b80aa4da12b628838a78dc79793f4d669ae1b CONFIGURE_COMMAND CC=gcc ./configure.sh UPDATE_COMMAND "" BUILD_COMMAND make From decccaf7ccaccebb7c34867230cab13629f21524 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 2 Aug 2018 15:43:35 +0200 Subject: [PATCH 508/723] test: Fix issue with virtio_block CMake script --- test/fs/integration/virtio_block/CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/fs/integration/virtio_block/CMakeLists.txt b/test/fs/integration/virtio_block/CMakeLists.txt index 28289bed15..f65b3dc95f 100644 --- a/test/fs/integration/virtio_block/CMakeLists.txt +++ b/test/fs/integration/virtio_block/CMakeLists.txt @@ -14,6 +14,10 @@ set(SOURCES service.cpp ) -set(DRIVERS virtioblk solo5blk) +if ("$ENV{PLATFORM}" STREQUAL "x86_solo5") + set(DRIVERS solo5blk) +else() + set(DRIVERS virtioblk) +endif() include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) From 868aff1f3c6a124715058571037575a83f9e176c Mon Sep 17 00:00:00 2001 From: John Zhang Date: Thu, 2 Aug 2018 23:35:15 +0800 Subject: [PATCH 509/723] refactor emplace back --- src/net/super_stack.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/net/super_stack.cpp b/src/net/super_stack.cpp index 95171a1b8d..0d24fdfd66 100644 --- a/src/net/super_stack.cpp +++ b/src/net/super_stack.cpp @@ -153,7 +153,7 @@ Super_stack::Super_stack() INFO("Network", "No registered network interfaces found"); for (size_t i = 0; i < hw::Devices::devices().size(); i++) { - ip4_stacks_.emplace_back(Stacks{}); + ip4_stacks_.emplace_back(); ip4_stacks_.back()[0] = nullptr; } } From f05bb4c32669fe3ca193ed0850ff5960cd853196 Mon Sep 17 00:00:00 2001 From: John Zhang Date: Fri, 3 Aug 2018 00:36:08 +0800 Subject: [PATCH 510/723] refactor emplace_back --- test/kernel/integration/fiber/service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/kernel/integration/fiber/service.cpp b/test/kernel/integration/fiber/service.cpp index a3aca32477..e8ce2f7993 100644 --- a/test/kernel/integration/fiber/service.cpp +++ b/test/kernel/integration/fiber/service.cpp @@ -180,7 +180,7 @@ void scheduler3 () { INFO("Scheduler", "Started. Initializing %i threads", N); for (int i = 0; i < N; i++) { - threads.emplace_back(Fiber{work}); + threads.emplace_back(work); } for (auto& thread : threads) { From ecb35bc656455c779ef5ebfb88442a029d8472ef Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 3 Aug 2018 10:12:45 +0200 Subject: [PATCH 511/723] x86: Protect symbols and strings --- src/crt/c_abi.c | 14 +++++++++----- src/kernel/elf.cpp | 15 +++++++++++++++ src/platform/x86_pc/os.cpp | 17 ++++++++++------- 3 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index f19974dc8a..9a7df03c1e 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -22,22 +22,26 @@ #define weak_alias(name, aliasname) _weak_alias (name, aliasname) #define _weak_alias(name, aliasname) \ extern __typeof (name) aliasname __attribute__ ((weak, alias (#name))); -#define HEAP_ALIGNMENT 63 void* __dso_handle; uint32_t _move_symbols(void* sym_loc) { + // re-align new symbol area to a page boundary + const intptr_t align_size = 4096 - ((uintptr_t) sym_loc & 4095); + sym_loc = (void*) ((uintptr_t) sym_loc + align_size); + extern char _ELF_SYM_START_; - /// read out size of symbols **before** moving them + // read out size of symbols **before** moving them extern int _get_elf_section_datasize(const void*); int elfsym_size = _get_elf_section_datasize(&_ELF_SYM_START_); - elfsym_size = (elfsym_size < HEAP_ALIGNMENT) ? HEAP_ALIGNMENT : elfsym_size; - /// move ELF symbols to safe area + // move ELF symbols to safe area extern void _move_elf_syms_location(const void*, void*); _move_elf_syms_location(&_ELF_SYM_START_, sym_loc); - return elfsym_size; + // increment elfsym_size to next page boundary + if (elfsym_size & 4095) elfsym_size += 4096 - (elfsym_size & 4095); + return align_size + elfsym_size; } void* __memcpy_chk(void* dest, const void* src, size_t len, size_t destlen) diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index 8edef083e2..219bb52b42 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -195,6 +195,7 @@ class ElfTables StrTab strtab; uint32_t checksum_syms; uint32_t checksum_strs; + friend void elf_protect_symbol_areas(); }; static ElfTables parser; @@ -437,3 +438,17 @@ void _init_elf_parser() parser.set(nullptr, 0, nullptr, 0, 0, 0); } } + +#ifdef ARCH_x86_64 +#include +void elf_protect_symbol_areas() +{ + const char* src = (const char*) parser.symtab.base; + ptrdiff_t size = &parser.strtab.base[parser.strtab.size] - src; + if (size & 4095) size += 4096 - (size & 4095); + INFO2("* Protecting syms %p to %p (size %#zx)\n", + src, src + size, size); + + os::mem::protect((uintptr_t) src, size, os::mem::Access::read); +} +#endif diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 3c08c8aa43..7aa30aeb75 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -37,7 +37,7 @@ #endif extern "C" void* get_cpu_esp(); -//extern "C" void __libc_init_array(); +extern "C" void kernel_sanity_checks(); extern uintptr_t _start; extern uintptr_t _end; extern uintptr_t _ELF_START_; @@ -96,10 +96,6 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) MYINFO("Stack: %p", get_cpu_esp()); MYINFO("Boot magic: 0x%x, addr: 0x%x", boot_magic, boot_addr); - // Call global ctors - PROFILE("Global constructors"); - //__libc_init_array(); - // PAGING // PROFILE("Enable paging"); extern void __arch_init_paging(); @@ -158,9 +154,16 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) memmap.assign_range({heap_begin_, heap_range_max_, "Dynamic memory", heap_usage }); +#ifdef ARCH_x86_64 + // memory protection on ELF symbols & strings + extern void elf_protect_symbol_areas(); + elf_protect_symbol_areas(); +#endif + MYINFO("Virtual memory map"); - for (const auto &i : memmap) - INFO2("%s",i.second.to_string().c_str()); + for (const auto& entry : memmap) + INFO2("%s", entry.second.to_string().c_str()); + kernel_sanity_checks(); PROFILE("Platform init"); extern void __platform_init(); From 4c4c4b67400cfc68e5e4e5e52b752025ea97a205 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 6 Aug 2018 10:54:56 +0200 Subject: [PATCH 512/723] ircd: Automate LiveUpdate example --- examples/IRCd/ircd/common.hpp | 2 -- examples/LiveUpdate/CMakeLists.txt | 14 ++++++++++++++ examples/LiveUpdate/ircd.motd | 12 ++++++++++++ examples/LiveUpdate/ircd.version | 0 examples/LiveUpdate/motd_array.h | 3 +++ examples/LiveUpdate/service.cpp | 14 ++------------ examples/LiveUpdate/update.sh | 5 +++++ 7 files changed, 36 insertions(+), 14 deletions(-) create mode 100644 examples/LiveUpdate/ircd.motd create mode 100644 examples/LiveUpdate/ircd.version create mode 100644 examples/LiveUpdate/motd_array.h diff --git a/examples/IRCd/ircd/common.hpp b/examples/IRCd/ircd/common.hpp index 9da1ecb027..89938e7f9d 100644 --- a/examples/IRCd/ircd/common.hpp +++ b/examples/IRCd/ircd/common.hpp @@ -1,8 +1,6 @@ #pragma once #include -#define IRC_SERVER_VERSION "0.3" - #define READQ_MAX 512 typedef uint32_t clindex_t; diff --git a/examples/LiveUpdate/CMakeLists.txt b/examples/LiveUpdate/CMakeLists.txt index b17ce73d9b..53e3e7c866 100644 --- a/examples/LiveUpdate/CMakeLists.txt +++ b/examples/LiveUpdate/CMakeLists.txt @@ -13,6 +13,11 @@ set(SERVICE_NAME "LiveUpdate example") # Name of your service binary set(BINARY "liveupdate") +# Custom version string of the IRC server +set(VERSION "v0.4 LiveUpdate demo") + +add_definitions(-DIRC_SERVER_VERSION="${VERSION}") + # Source files to be linked with OS library parts to form bootable image set(SOURCES service.cpp @@ -55,3 +60,12 @@ set(LIBRARIES include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diskbuilder(drive) + +add_custom_command( + OUTPUT motd.h + COMMAND xxd -i < ${CMAKE_CURRENT_SOURCE_DIR}/ircd.motd > motd.h + DEPENDS ircd.motd + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} +) +add_custom_target(motd DEPENDS motd.h) +add_dependencies(service motd) diff --git a/examples/LiveUpdate/ircd.motd b/examples/LiveUpdate/ircd.motd new file mode 100644 index 0000000000..43f146eb1a --- /dev/null +++ b/examples/LiveUpdate/ircd.motd @@ -0,0 +1,12 @@ +.-') _ _ .-') _ ('-. .-') +( OO ) ) ( ( OO) ) _( OO) ( OO ). +,-.-') ,--./ ,--,' .-----. ,--. ,--. ,--. \ .'_ (,------. .-'),-----. (_)---\_) +| |OO)| \ | |\ ' .--./ | |.-') | | | | ,`'--..._) | .---' ( OO' .-. '/ _ | +| | \| \| | )| |('-. | | OO )| | | .-') | | \ ' | | / | | | |\ :` `. +| |(_/| . |//_) |OO )| |`-' || |_|( OO )| | ' |(| '--. \_) | |\| | '..`''.) +,| |_.'| |\ | || |`-'|(| '---.'| | | `-' /| | / : | .--' \ | | | |.-._) \ +(_| | | | \ |(_' '--'\ | |(' '-'(_.-' | '--' / | `---. `' '-' '\ / +`--' `--' `--' `-----' `------' `-----' `-------' `------' `-----' `-----' + +Test2 +Test3 diff --git a/examples/LiveUpdate/ircd.version b/examples/LiveUpdate/ircd.version new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/LiveUpdate/motd_array.h b/examples/LiveUpdate/motd_array.h new file mode 100644 index 0000000000..7faf8f3ca6 --- /dev/null +++ b/examples/LiveUpdate/motd_array.h @@ -0,0 +1,3 @@ +static const char motd_array[] = { +#include "build/motd.h" +, 0x0}; diff --git a/examples/LiveUpdate/service.cpp b/examples/LiveUpdate/service.cpp index 3b04128483..179b4dfe9a 100644 --- a/examples/LiveUpdate/service.cpp +++ b/examples/LiveUpdate/service.cpp @@ -18,7 +18,7 @@ #include #include #include -#include +#include "motd_array.h" #define USE_STACK_SAMPLING #define PERIOD_SECS 4 @@ -34,17 +34,7 @@ void Service::start() ircd = IrcServer::from_config(); ircd->set_motd([] () -> const std::string& { - static const std::string motd = R"M0TDT3XT( - .-') _ _ .-') _ ('-. .-') - ( OO ) ) ( ( OO) ) _( OO) ( OO ). - ,-.-') ,--./ ,--,' .-----. ,--. ,--. ,--. \ .'_ (,------. .-'),-----. (_)---\_) - | |OO)| \ | |\ ' .--./ | |.-') | | | | ,`'--..._) | .---' ( OO' .-. '/ _ | - | | \| \| | )| |('-. | | OO )| | | .-') | | \ ' | | / | | | |\ :` `. - | |(_/| . |//_) |OO )| |`-' || |_|( OO )| | ' |(| '--. \_) | |\| | '..`''.) - ,| |_.'| |\ | || |`-'|(| '---.'| | | `-' /| | / : | .--' \ | | | |.-._) \ -(_| | | | \ |(_' '--'\ | |(' '-'(_.-' | '--' / | `---. `' '-' '\ / - `--' `--' `--' `-----' `------' `-----' `-------' `------' `-----' `-----' -)M0TDT3XT"; + static std::string motd(motd_array); return motd; }); // motd spam diff --git a/examples/LiveUpdate/update.sh b/examples/LiveUpdate/update.sh index 3fbea18c19..bdb99a1625 100755 --- a/examples/LiveUpdate/update.sh +++ b/examples/LiveUpdate/update.sh @@ -1,2 +1,7 @@ #!/bin/bash +mkdir -p build +pushd build +cmake .. +make -j8 +popd dd if=build/liveupdate > /dev/tcp/10.0.0.42/666 From e32c2a163f98a079ceb11428da7cf35207768d19 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 6 Aug 2018 11:54:39 +0200 Subject: [PATCH 513/723] x86: Fix syscal_entry asm --- src/arch/x86_64/__syscall_entry.asm | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/arch/x86_64/__syscall_entry.asm b/src/arch/x86_64/__syscall_entry.asm index 909110b768..d1146d514b 100644 --- a/src/arch/x86_64/__syscall_entry.asm +++ b/src/arch/x86_64/__syscall_entry.asm @@ -61,12 +61,10 @@ extern kprintf section .bss temp_stack: resb stack_size - -section .data: temp_old_stack: - dq 0 + resq 1 temp_rcx: - dq 0 + resq 1 section .text __syscall_entry: From 071b565407ad252314f3ae521ff679c057cb8999 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Mon, 6 Aug 2018 14:12:24 +0200 Subject: [PATCH 514/723] alloc: add os namespace to mem::buddy to avoid ambiguity --- api/util/alloc_buddy.hpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index 969fcbf9d6..0f457c1fe8 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -1,3 +1,4 @@ + // This file is a part of the IncludeOS unikernel - www.includeos.org // // Copyright 2018 IncludeOS AS, Oslo, Norway @@ -33,13 +34,12 @@ namespace std::pmr { #include #include -#include // // Tree node flags // -namespace mem::buddy { +namespace os::mem::buddy { enum class Flags : uint8_t { free = 0, taken = 1, @@ -58,13 +58,13 @@ namespace util { }; template<> - struct enable_bitmask_ops { + struct enable_bitmask_ops { using type = uint8_t; static constexpr bool enable = true; }; } -namespace mem::buddy { +namespace os::mem::buddy { using namespace util::literals; using namespace util::bitops; From a953a96efb8eb6c29e6f7c80c906b611d67448d8 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 7 Aug 2018 14:13:34 +0200 Subject: [PATCH 515/723] x86: Protect pagetables, symbols, move statman --- api/arch/x86/paging.hpp | 13 +++++++++++++ api/util/crc32.hpp | 7 +++++++ src/arch/x86_64/paging.cpp | 33 +++++++++++++++++++++++++++++++++ src/fs/memdisk.cpp | 2 +- src/kernel/elf.cpp | 22 +++++++++------------- src/kernel/multiboot.cpp | 4 ++-- src/platform/x86_pc/os.cpp | 30 ++++++++++++------------------ 7 files changed, 77 insertions(+), 34 deletions(-) diff --git a/api/arch/x86/paging.hpp b/api/arch/x86/paging.hpp index f14480bcd8..116ee3b66c 100644 --- a/api/arch/x86/paging.hpp +++ b/api/arch/x86/paging.hpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -806,6 +807,16 @@ class Page_table { { return tbl_.at(i); } + void traverse(delegate callback) + { + callback(this, sizeof(Page_table)); + for (auto& ent : tbl_) + if(is_page_dir(ent)) { + auto* pdir = page_dir(&ent); + pdir->traverse(callback); + } + } + private: alignas(Arch::min_pagesize) std::array tbl_ {}; const uintptr_t linear_addr_start_ = 0; @@ -909,6 +920,8 @@ inline Map Pml1::map_r(Map req) /** Invalidate page (e.g. flush TLB entry) **/ void invalidate(void *pageaddr); +template <> +inline void Pml1::traverse(delegate) {} } // namespace paging } // namespace x86 diff --git a/api/util/crc32.hpp b/api/util/crc32.hpp index e76a905f73..6fbb30cdef 100644 --- a/api/util/crc32.hpp +++ b/api/util/crc32.hpp @@ -93,4 +93,11 @@ inline uint32_t crc32(const void* buf, size_t len) /** Intel (iSCSI) aka CRC32-C with hardware support **/ extern uint32_t crc32_fast(const void* buf, size_t len); +/** Software-only CRC32-C **/ +inline uint32_t crc32c(const void* buf, size_t len) +{ + extern uint32_t crc32c_sw(uint32_t, const char*, size_t); + return ~crc32c_sw(0xFFFFFFFF, (const char*) buf, len); +} + #endif diff --git a/src/arch/x86_64/paging.cpp b/src/arch/x86_64/paging.cpp index fedd27ef9e..74deebfa7d 100644 --- a/src/arch/x86_64/paging.cpp +++ b/src/arch/x86_64/paging.cpp @@ -40,6 +40,7 @@ using Flags = x86::paging::Flags; using Pml4 = x86::paging::Pml4; static void allow_executable(); +static void protect_pagetables_once(); // must be public symbols because of a unittest extern char _TEXT_START_; @@ -113,6 +114,11 @@ void __arch_init_paging() { Expects(! __pml4->has_flag((uintptr_t)__exec_begin, Flags::no_exec)); Expects(__pml4->has_flag((uintptr_t)__exec_begin, Flags::present)); + extern void elf_protect_symbol_areas(); + elf_protect_symbol_areas(); + + // hack to prevent see who overwrites the pagetables + protect_pagetables_once(); INFO2("* Passing page tables to CPU"); extern void __x86_init_paging(void*); @@ -378,3 +384,30 @@ void allow_executable() os::mem::map(m, "ELF .text"); } + +void protect_pagetables_once() +{ + struct ptinfo { + void* addr; + size_t len; + }; + std::vector table_entries; + + __pml4->traverse( + [&table_entries] (void* ptr, size_t len) { + table_entries.push_back({ptr, len}); + }); + + + for (auto it = table_entries.rbegin(); it != table_entries.rend(); ++it) + { + const auto& entry = *it; + const x86::paging::Map m { + (uintptr_t) entry.addr, x86::paging::any_addr, + x86::paging::Flags::present, static_cast(entry.len) + }; + assert(m); + MEM_PRINT("Protecting table: %p, size %zu\n", entry.addr, entry.len); + __pml4->map_r(m); + } +} diff --git a/src/fs/memdisk.cpp b/src/fs/memdisk.cpp index 7d8fb28855..f6af4f2db3 100644 --- a/src/fs/memdisk.cpp +++ b/src/fs/memdisk.cpp @@ -32,7 +32,7 @@ namespace fs { MemDisk::MemDisk() noexcept : MemDisk(&_DISK_START_, &_DISK_END_) { - INFO("Memdisk", "Initializing"); + INFO("Memdisk", "Initializing start=%p end=%p", image_start_, image_end_); } MemDisk::MemDisk(const char* start, const char* end) noexcept diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index 219bb52b42..2d5dd18320 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -162,14 +162,16 @@ class ElfTables } bool verify_symbols() const { + printf("ELF verification from %p to %p (Syms=%#x, Strs=%#x)\n", + symtab.base, strtab.base + strtab.size, checksum_syms, checksum_strs); uint32_t csum = - crc32_fast(symtab.base, symtab.entries * sizeof(ElfSym)); + crc32c(symtab.base, symtab.entries * sizeof(ElfSym)); if (csum != checksum_syms) { printf("ELF symbol tables checksum failed! " "(%08x vs %08x)\n", csum, checksum_syms); return false; } - csum = crc32_fast(strtab.base, strtab.size); + csum = crc32c(strtab.base, strtab.size); if (csum != checksum_strs) { printf("ELF string tables checksum failed! " "(%08x vs %08x)\n", csum, checksum_strs); @@ -193,8 +195,8 @@ class ElfTables SymTab symtab; StrTab strtab; - uint32_t checksum_syms; - uint32_t checksum_strs; + uint32_t checksum_syms = 0; + uint32_t checksum_strs = 0; friend void elf_protect_symbol_areas(); }; static ElfTables parser; @@ -363,12 +365,6 @@ int _get_elf_section_datasize(const void* location) return hdr.symtab_entries * sizeof(ElfSym) + hdr.strtab_size; } -// we unfortunately cannot call crc32_fast directly, as it changes .bss -extern uint32_t crc32c_sw(uint32_t, const char*, size_t); -inline uint32_t crc32c(const void* buffer, size_t len) { - return ~crc32c_sw(0xFFFFFFFF, (const char*) buffer, len); -} - extern "C" void _move_elf_syms_location(const void* location, void* new_location) { @@ -443,12 +439,12 @@ void _init_elf_parser() #include void elf_protect_symbol_areas() { - const char* src = (const char*) parser.symtab.base; + char* src = (char*) parser.symtab.base; ptrdiff_t size = &parser.strtab.base[parser.strtab.size] - src; if (size & 4095) size += 4096 - (size & 4095); INFO2("* Protecting syms %p to %p (size %#zx)\n", - src, src + size, size); + src, &parser.strtab.base[parser.strtab.size], size); - os::mem::protect((uintptr_t) src, size, os::mem::Access::read); + //os::mem::protect((uintptr_t) src, size, os::mem::Access::read); } #endif diff --git a/src/kernel/multiboot.cpp b/src/kernel/multiboot.cpp index 09ef88bf60..4e0ca579cc 100644 --- a/src/kernel/multiboot.cpp +++ b/src/kernel/multiboot.cpp @@ -158,8 +158,8 @@ void OS::multiboot(uint32_t boot_addr) if (not (map.type & MULTIBOOT_MEMORY_AVAILABLE)) { if (util::bits::is_aligned<4_KiB>(map.addr)) { - os::mem::map({addr, addr, os::mem::Access::read | os::mem::Access::write, size}, - "Reserved (Multiboot)"); + //os::mem::map({addr, addr, os::mem::Access::read | os::mem::Access::write, size}, + // "Reserved (Multiboot)"); continue; } diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 7aa30aeb75..621f699e0e 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -82,6 +82,7 @@ void OS::default_stdout(const char* str, const size_t len) void OS::start(uint32_t boot_magic, uint32_t boot_addr) { + PROFILE("OS::start"); OS::cmdline = Service::binary_name(); // Initialize stdout handlers @@ -89,7 +90,6 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) OS::add_stdout(&OS::default_stdout); } - PROFILE("OS::start"); // Print a fancy header CAPTION("#include // Literally"); @@ -119,11 +119,7 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) // Give the rest of physical memory to heap OS::heap_max_ = OS::memory_end_ - 1; - - /// STATMAN /// - PROFILE("Statman"); - /// initialize on page 9, 8 pages in size - Statman::get().init(0x8000, 0x8000); + assert(heap_begin() != 0x0 and OS::heap_max_ != 0x0); PROFILE("Memory map"); // Assign memory ranges used by the kernel @@ -136,17 +132,15 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) * TODO: Map binary parts using mem::map instead of assigning ranges directly * e.g. the .text segment is now mapped individually by __arch_init_paging */ - memmap.assign_range({0x1000, 0x6fff, "Pagetables"}); + memmap.assign_range({ 0x1000, 0x7fff, "Pagetables"}); memmap.assign_range({0x10000, 0x9d3ff, "Stack"}); #elif defined(ARCH_i686) memmap.assign_range({0x10000, 0x9d3ff, "Stack"}); #endif + memmap.assign_range({(uintptr_t)&_end, heap_begin()-1, + "Symbols & strings"}); - assert(heap_begin() != 0x0 and OS::heap_max_ != 0x0); - // @note for security we don't want to expose this - memmap.assign_range({(uintptr_t)&_end, heap_begin() - 1, - "Pre-heap"}); - + // heap (physical) area uintptr_t span_max = std::numeric_limits::max(); uintptr_t heap_range_max_ = std::min(span_max, OS::heap_max_); @@ -154,17 +148,17 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) memmap.assign_range({heap_begin_, heap_range_max_, "Dynamic memory", heap_usage }); -#ifdef ARCH_x86_64 - // memory protection on ELF symbols & strings - extern void elf_protect_symbol_areas(); - elf_protect_symbol_areas(); -#endif - + kernel_sanity_checks(); MYINFO("Virtual memory map"); for (const auto& entry : memmap) INFO2("%s", entry.second.to_string().c_str()); kernel_sanity_checks(); + /// STATMAN /// + PROFILE("Statman"); + /// initialize on page 9, 8 pages in size + Statman::get().init(0x8000, 0x8000); + PROFILE("Platform init"); extern void __platform_init(); __platform_init(); From f1c2d5f3afcc6038839e83877925d4d86dd8346c Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 7 Aug 2018 14:22:02 +0200 Subject: [PATCH 516/723] memory: add system C++ allocator --- api/kernel/memory.hpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/api/kernel/memory.hpp b/api/kernel/memory.hpp index dc08190963..13f6fb8885 100644 --- a/api/kernel/memory.hpp +++ b/api/kernel/memory.hpp @@ -22,6 +22,7 @@ #include #include #include +#include #include namespace os { @@ -35,6 +36,11 @@ namespace mem { execute = 4 }; + /** Default system allocator **/ + using Allocator = buddy::Alloc; + + Allocator& allocator(); + /** Get bitfield with bit set for each supported page size */ uintptr_t supported_page_sizes(); From 5b542380448d1e145081df465476701b8058018c Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 7 Aug 2018 14:23:24 +0200 Subject: [PATCH 517/723] buddy: allow pool sizes larger than physical memory --- api/util/alloc_buddy.hpp | 84 ++++++++++++++++++++++++++++++---------- 1 file changed, 64 insertions(+), 20 deletions(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index 0f457c1fe8..99558406d6 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -46,7 +46,7 @@ namespace os::mem::buddy { left_used = 2, right_used = 4, left_full = 8, - right_full = 16, + right_full = 16 }; } @@ -86,33 +86,62 @@ namespace os::mem::buddy { return ((pool_size / min_size) * 2) - 1; } + static constexpr Size_t overhead(Size_t pool_size) { + using namespace util; + auto nodes_size = node_count(pool_size) * sizeof(Node_t); + auto overhead_ = bits::roundto(min_size, sizeof(Alloc) + nodes_size); + return overhead_; + } - /** Total allocatable memory in an allocator created in a bufsize buffer **/ + /** + * Indicate if the allocator should manage a power of 2 larger than- or + * smaller than available memory. + **/ + enum class Policy { + underbook, overbook + }; + + /** + * Total allocatable memory in an allocator created in a bufsize buffer. + * If the policy is overbook, we compute the next power of two above bufsize, + * otherwise the next power of two below. + **/ + template static Size_t pool_size(Size_t bufsize){ using namespace util; - // Find closest power of two below bufsize - auto pool_size_ = bits::keeplast(bufsize - 1); - auto unalloc = bufsize - pool_size_; - auto node_size = node_count(pool_size_) * sizeof(Node_t); - auto overhead = bits::roundto(min_size, sizeof(Alloc) + node_size); + auto pool_size = 0; + // Find closest usable power of two depending on policy + if constexpr (P == Policy::overbook) { + return bits::next_pow2(bufsize); + } else { + pool_size = bits::keeplast(bufsize - 1); // -1 starts the recursion + } + + auto unalloc = bufsize - pool_size; + auto overhead = Alloc::overhead(pool_size); // If bufsize == overhead + pow2, and overhead was too small to fit alloc // Try the next power of two recursively if(unalloc < overhead) - return pool_size(pool_size_); + return Alloc::pool_size(pool_size); - auto free = bufsize - overhead; - auto pool_size = bits::keeplast(free); - return pool_size; + auto free = bufsize - overhead; + return bits::keeplast(free); } /** Required total bufsize to manage pool_size memory **/ + template static Size_t required_size(Size_t pool_size) { using namespace util; + + if constexpr (P == Policy::overbook) { + return overhead(pool_size); + } + return bits::roundto(min_size, pool_size) - + bits::roundto(min_size, sizeof(Alloc) + node_count(pool_size) * sizeof(Node_t)); + + overhead(pool_size); } Index_t node_count() const noexcept { @@ -136,10 +165,15 @@ namespace os::mem::buddy { return min_size << (tree_height() - 1); } + /** Theoretically allocatable capacity */ Size_t pool_size() const noexcept { return pool_size_; } + Size_t bytes_unavailable() const noexcept { + return pool_size_ - (addr_limit_ - start_addr_); + } + Size_t bytes_used() const noexcept { return bytes_used_; } @@ -149,7 +183,7 @@ namespace os::mem::buddy { } Size_t bytes_free() const noexcept { - return pool_size_ - bytes_used_; + return pool_size_ - bytes_used_ - bytes_unavailable(); } Size_t bytes_free_r() const noexcept { @@ -186,7 +220,7 @@ namespace os::mem::buddy { Alloc(void* start, Size_t bufsize, Size_t pool_size) : nodes_{(Node_t*)start, node_count(pool_size)}, - start_addr_{util::bits::roundto(min_size, (Addr_t)start + node_count(pool_size))}, + start_addr_{util::bits::roundto(min_size, (Addr_t)start + node_count(pool_size))}, addr_limit_{reinterpret_cast(start) + bufsize}, pool_size_{pool_size} { @@ -194,11 +228,15 @@ namespace os::mem::buddy { Expects(bits::is_pow2(pool_size_)); Expects(pool_size_ >= min_size); Expects(bits::is_aligned(start_addr_)); - Ensures(pool_size_ + nodes_.size() * sizeof(Node_t) <= bufsize); + Ensures(bufsize >= overhead(pool_size)); // Initialize nodes memset(nodes_.data(), 0, nodes_.size() * sizeof(Node_arr::element_type)); } + /** + * Create an allocator + **/ + template static Alloc* create(void* addr, Size_t bufsize) { using namespace util; Size_t pool_size_ = pool_size(bufsize); @@ -206,7 +244,9 @@ namespace os::mem::buddy { // Placement new an allocator on addr, passing in the rest of memory auto* alloc_begin = (char*)addr + sizeof(Alloc); - auto* alloc = new (addr) Alloc(alloc_begin, bufsize, pool_size_); + auto* alloc = new (addr) Alloc(alloc_begin, + bufsize + sizeof(Alloc), + pool_size_); return alloc; } @@ -269,12 +309,19 @@ namespace os::mem::buddy { alloc_tracker(Track::start); auto res = node.allocate(sz); + + // For overbooking allocator, allow unusable memory to gradually become + // marked as allocated, without actually handing it out. + if (UNLIKELY(res + size > addr_limit_)) + return 0; + if (res) bytes_used_ += sz; return reinterpret_cast(res); } void deallocate(void* addr, Size_t size) { auto sz = size ? chunksize(size) : 0; + Expects(reinterpret_cast(addr) + size < addr_limit_); auto res = root().deallocate((Addr_t)addr, sz); Expects(not size or res == sz); bytes_used_ -= res; @@ -427,10 +474,6 @@ namespace os::mem::buddy { Expects(sz <= my_size_); Expects(! is_taken()); - if (is_taken()) { - return 0; - } - // Allocate self if possible if (sz == my_size_) { if (is_free()) { @@ -656,6 +699,7 @@ namespace os::mem::buddy { Node_arr nodes_; const uintptr_t start_addr_ = 0; + const uintptr_t addr_limit_ = 0; const Size_t pool_size_ = min_size; Size_t bytes_used_ = 0; }; From c1e737501175197be6607e3b80073e8c274ca3b3 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 7 Aug 2018 14:25:02 +0200 Subject: [PATCH 518/723] brk: remove recursion --- src/musl/brk.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/musl/brk.cpp b/src/musl/brk.cpp index 8b36126e68..a3e9cadf6b 100644 --- a/src/musl/brk.cpp +++ b/src/musl/brk.cpp @@ -26,7 +26,7 @@ size_t brk_bytes_used() { } size_t brk_bytes_free() { - return __brk_max - brk_bytes_free(); + return __brk_max - brk_bytes_used(); } static uintptr_t sys_brk(void* addr) From 5c8c1a7ebc5e517bcacdebd4045e7a321443a129 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 7 Aug 2018 14:26:07 +0200 Subject: [PATCH 519/723] mmap: fix return value, use os:mem::allocator --- src/musl/mmap.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 4f0850a18b..9bc73c40d8 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -4,11 +4,17 @@ #include #include #include +#include #include -using Alloc = mem::buddy::Alloc<>; +using Alloc = os::mem::Allocator; static Alloc* alloc; +Alloc& os::mem::allocator() { + Expects(alloc); + return *alloc; +} + uintptr_t __init_mmap(uintptr_t addr_begin) { auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); @@ -39,7 +45,7 @@ size_t mmap_bytes_free() { return alloc->bytes_free(); } -uintptr_t mmap_allocation_end(){ +uintptr_t mmap_allocation_end() { return alloc->highest_used(); } @@ -56,7 +62,12 @@ static void* sys_mmap(void *addr, size_t length, int /*prot*/, int /*flags*/, return MAP_FAILED; } - return kalloc(length); + auto* res = kalloc(length); + + if (UNLIKELY(res == nullptr)) + return MAP_FAILED; + + return res; } extern "C" From 6e4790351bae27cda560ca36a54867012288a0e6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 7 Aug 2018 14:32:47 +0200 Subject: [PATCH 520/723] x86: Disable fixed protection of pagetables --- src/arch/x86_64/paging.cpp | 2 +- src/kernel/elf.cpp | 4 ++-- src/kernel/multiboot.cpp | 4 ++-- src/platform/x86_pc/os.cpp | 3 +-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/arch/x86_64/paging.cpp b/src/arch/x86_64/paging.cpp index 74deebfa7d..65cbabc398 100644 --- a/src/arch/x86_64/paging.cpp +++ b/src/arch/x86_64/paging.cpp @@ -118,7 +118,7 @@ void __arch_init_paging() { elf_protect_symbol_areas(); // hack to prevent see who overwrites the pagetables - protect_pagetables_once(); + //protect_pagetables_once(); INFO2("* Passing page tables to CPU"); extern void __x86_init_paging(void*); diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index 2d5dd18320..c95e6cd666 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -165,13 +165,13 @@ class ElfTables printf("ELF verification from %p to %p (Syms=%#x, Strs=%#x)\n", symtab.base, strtab.base + strtab.size, checksum_syms, checksum_strs); uint32_t csum = - crc32c(symtab.base, symtab.entries * sizeof(ElfSym)); + crc32_fast(symtab.base, symtab.entries * sizeof(ElfSym)); if (csum != checksum_syms) { printf("ELF symbol tables checksum failed! " "(%08x vs %08x)\n", csum, checksum_syms); return false; } - csum = crc32c(strtab.base, strtab.size); + csum = crc32_fast(strtab.base, strtab.size); if (csum != checksum_strs) { printf("ELF string tables checksum failed! " "(%08x vs %08x)\n", csum, checksum_strs); diff --git a/src/kernel/multiboot.cpp b/src/kernel/multiboot.cpp index 4e0ca579cc..09ef88bf60 100644 --- a/src/kernel/multiboot.cpp +++ b/src/kernel/multiboot.cpp @@ -158,8 +158,8 @@ void OS::multiboot(uint32_t boot_addr) if (not (map.type & MULTIBOOT_MEMORY_AVAILABLE)) { if (util::bits::is_aligned<4_KiB>(map.addr)) { - //os::mem::map({addr, addr, os::mem::Access::read | os::mem::Access::write, size}, - // "Reserved (Multiboot)"); + os::mem::map({addr, addr, os::mem::Access::read | os::mem::Access::write, size}, + "Reserved (Multiboot)"); continue; } diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 621f699e0e..7715511127 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -148,11 +148,10 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) memmap.assign_range({heap_begin_, heap_range_max_, "Dynamic memory", heap_usage }); - kernel_sanity_checks(); MYINFO("Virtual memory map"); for (const auto& entry : memmap) INFO2("%s", entry.second.to_string().c_str()); - kernel_sanity_checks(); + //kernel_sanity_checks(); /// STATMAN /// PROFILE("Statman"); From 5f6aca95535d2fd7c6ad069c9e564b43b63cdd02 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 7 Aug 2018 14:55:34 +0200 Subject: [PATCH 521/723] mmap: Return MAP_FAILED on failed kallocs --- src/musl/mmap.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 694d194333..71bff82b7d 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -55,7 +55,9 @@ static void* sys_mmap(void *addr, size_t length, int /*prot*/, int /*flags*/, return MAP_FAILED; } - return kalloc(length); + auto* ptr = kalloc(length); + if (ptr == nullptr) return MAP_FAILED; + return ptr; } extern "C" From e383c471f2ae82dcee8c550de13d63d76d000654 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 7 Aug 2018 20:39:54 +0200 Subject: [PATCH 522/723] paging: fix bug in protect + regression test --- api/arch/x86/paging.hpp | 2 +- test/kernel/unit/memory.cpp | 68 +++++++++++++++++++++++++++++++++++-- 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/api/arch/x86/paging.hpp b/api/arch/x86/paging.hpp index f14480bcd8..3ccd6f9508 100644 --- a/api/arch/x86/paging.hpp +++ b/api/arch/x86/paging.hpp @@ -754,7 +754,7 @@ class Page_table { { auto* ent = entry(req.lin + res.size); - Map sub {req.lin + res.size, req.phys + res.size, req.flags, + Map sub {req.lin + res.size, req.phys == any_addr ? any_addr : req.phys + res.size, req.flags, req.size - res.size, req.page_sizes}; res += map_entry_r(ent, sub); diff --git a/test/kernel/unit/memory.cpp b/test/kernel/unit/memory.cpp index def0968bd7..ec8c8b2824 100644 --- a/test/kernel/unit/memory.cpp +++ b/test/kernel/unit/memory.cpp @@ -127,7 +127,7 @@ class Default_paging { __arch_init_paging(); } -private: + static void clear_paging() { using namespace x86::paging; MYINFO("Clearing default paging \n"); @@ -372,7 +372,6 @@ SETUP ("Assuming a default page table setup") EXPECT(sizes_pre[1] >= sizes_post[1]); // Is now likely 2_MiB or 4_KiB EXPECT(sizes_pre[2] > sizes_post[2]); // Must be the new size EXPECT(sizes_pre[3] == sizes_post[3]); // Can't have changed, 1 GiB above - } } } @@ -380,6 +379,7 @@ SETUP ("Assuming a default page table setup") CASE("os::mem::protect try to break stuff"){ using namespace util::literals; auto init_access = mem::Access::none; + Default_paging::clear_paging(); EXPECT(__pml4 == nullptr); __pml4 = new x86::paging::Pml4(0); @@ -431,5 +431,67 @@ CASE("os::mem::protect try to break stuff"){ EXPECT(__pml4->bytes_allocated() == initial_use); } MYINFO("Allocated bytes at end: %zi \n", __pml4->bytes_allocated()); - delete __pml4; + Default_paging::clear_paging(); +} + + +CASE("os::mem::protect verify consistency"){ + using namespace util::literals; + auto init_access = mem::Access::none; + + if (__pml4 != nullptr) { + printf("NOT NULL\n"); + } + EXPECT(__pml4 == nullptr); + + __pml4 = new x86::paging::Pml4(0); + EXPECT(__pml4->is_empty()); + + auto initial_use = __pml4->bytes_allocated(); + MYINFO("Initial memory use: %zi \n", initial_use); + + + mem::Map req {6_GiB, 3_GiB, mem::Access::read | mem::Access::write, 300_MiB}; + auto res = mem::map(req); + EXPECT(res); + EXPECT(res.flags == (mem::Access::write | mem::Access::read)); + EXPECT(res.lin == req.lin); + EXPECT(res.phys == req.phys); + + auto prot_offs = 1_MiB; + auto prot_begin = 6_GiB + prot_offs; + auto prot_size = 1043_KiB; + auto diff_phys = req.lin - req.phys; + auto new_flags = mem::Access::read; + + // Write-protect + auto prot = mem::protect(prot_begin, prot_size, new_flags); + EXPECT(prot); + EXPECT(prot.flags == new_flags); + + // Verify each page + for (auto i = prot_begin; i < prot_begin + prot_size; i += mem::min_psize()) { + EXPECT(mem::virt_to_phys(i) == i - diff_phys); + auto flags = mem::flags(i); + EXPECT(flags == new_flags); + } + + auto memuse_after_prot = __pml4->bytes_allocated(); + EXPECT(__pml4->bytes_allocated() > initial_use); + + // Protect with different flags + new_flags = mem::Access::read | mem::Access::write | mem::Access::execute; + auto prot2 = mem::protect(prot_begin, prot_size, new_flags); + EXPECT(prot2); + + // Verify each page + for (auto i = prot_begin; i < prot_begin + prot_size; i += mem::min_psize()) { + EXPECT(mem::virt_to_phys(i) == i - diff_phys); + auto flags = mem::flags(i); + EXPECT(flags == new_flags); + } + + EXPECT(__pml4->bytes_allocated() == memuse_after_prot); + + Default_paging::clear_paging(); } From e01d87ac73b81ac6235be0a9b95a9653433d1924 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 8 Aug 2018 10:19:56 +0200 Subject: [PATCH 523/723] Merge with brokenness branch --- src/arch/x86_64/paging.cpp | 2 +- src/kernel/elf.cpp | 2 +- src/platform/x86_pc/os.cpp | 21 +++++++++++---------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/arch/x86_64/paging.cpp b/src/arch/x86_64/paging.cpp index 65cbabc398..30fc629c9d 100644 --- a/src/arch/x86_64/paging.cpp +++ b/src/arch/x86_64/paging.cpp @@ -117,7 +117,7 @@ void __arch_init_paging() { extern void elf_protect_symbol_areas(); elf_protect_symbol_areas(); - // hack to prevent see who overwrites the pagetables + // hack to see who overwrites the pagetables //protect_pagetables_once(); INFO2("* Passing page tables to CPU"); diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index c95e6cd666..379ca1800f 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -442,7 +442,7 @@ void elf_protect_symbol_areas() char* src = (char*) parser.symtab.base; ptrdiff_t size = &parser.strtab.base[parser.strtab.size] - src; if (size & 4095) size += 4096 - (size & 4095); - INFO2("* Protecting syms %p to %p (size %#zx)\n", + INFO2("* Protecting syms %p to %p (size %#zx)", src, &parser.strtab.base[parser.strtab.size], size); //os::mem::protect((uintptr_t) src, size, os::mem::Access::read); diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 7715511127..ceb38a6dd6 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -124,34 +125,34 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) PROFILE("Memory map"); // Assign memory ranges used by the kernel auto& memmap = memory_map(); - MYINFO("Assigning fixed memory ranges (Memory map)"); + INFO2("Assigning fixed memory ranges (Memory map)"); + // protect symbols early on (the calculation is complex so not doing it here) + memmap.assign_range({(uintptr_t)&_end, heap_begin()-1, + "Symbols & strings"}); + extern void elf_protect_symbol_areas(); + elf_protect_symbol_areas(); memmap.assign_range({0x8000, 0xffff, "Statman"}); #if defined(ARCH_x86_64) - /** - * TODO: Map binary parts using mem::map instead of assigning ranges directly - * e.g. the .text segment is now mapped individually by __arch_init_paging - */ - memmap.assign_range({ 0x1000, 0x7fff, "Pagetables"}); + // protect the basic pagetable used by LiveUpdate and any other + // systems that need to exit long/protected mode + os::mem::map({0x1000, 0x1000, os::mem::Access::read, 0x7000}, "Page tables"); memmap.assign_range({0x10000, 0x9d3ff, "Stack"}); #elif defined(ARCH_i686) memmap.assign_range({0x10000, 0x9d3ff, "Stack"}); #endif - memmap.assign_range({(uintptr_t)&_end, heap_begin()-1, - "Symbols & strings"}); // heap (physical) area uintptr_t span_max = std::numeric_limits::max(); uintptr_t heap_range_max_ = std::min(span_max, OS::heap_max_); - MYINFO("Assigning heap 0x%zx -> 0x%zx", heap_begin_, heap_range_max_); + INFO2("* Assigning heap 0x%zx -> 0x%zx", heap_begin_, heap_range_max_); memmap.assign_range({heap_begin_, heap_range_max_, "Dynamic memory", heap_usage }); MYINFO("Virtual memory map"); for (const auto& entry : memmap) INFO2("%s", entry.second.to_string().c_str()); - //kernel_sanity_checks(); /// STATMAN /// PROFILE("Statman"); From a9d57d70046cd77610d4d62f1ba2be1e02bb82d1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 8 Aug 2018 10:53:02 +0200 Subject: [PATCH 524/723] icmp6: Make test more likely to succeed --- test/net/integration/icmp6/test.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test/net/integration/icmp6/test.py b/test/net/integration/icmp6/test.py index 68300c8775..30958c0426 100755 --- a/test/net/integration/icmp6/test.py +++ b/test/net/integration/icmp6/test.py @@ -37,21 +37,25 @@ def start_icmp_test(trigger_line): "Destination: fe80:0:0:0:e823:fcff:fef4:85bd" in output_data and \ "Type: ECHO REPLY (129)" in output_data and \ "Code: DEFAULT (0)" in output_data and \ - "Checksum: " in output_data and \ - "Data: INCLUDEOS12345ABCDEFGHIJKLMNOPQRSTUVWXYZ12345678" in output_data: + "Checksum: " in output_data: + #"Data: INCLUDEOS12345ABCDEFGHIJKLMNOPQRSTUVWXYZ12345678" in output_data: num_successes += 1 print color.INFO(""), "Ping test succeeded" else: print color.FAIL(""), "Ping test FAILED" + vm.exit(1, 666) - if num_successes == 1: + if num_successes == 2: vm.exit(0, " All ICMP tests succeeded. Process returned 0 exit status") - else: - num_fails = 1 - num_successes - res = " " + str(num_fails) + " ICMP6 test(s) failed" - vm.exit(1, res) + +def data_line(text): + global num_successes + num_successes += 1 + if num_successes == 2: + vm.exit(0, " All ICMP tests succeeded. Process returned 0 exit status") vm.on_output("Service IPv4 address: 10.0.0.52, IPv6 address: fe80:0:0:0:e823:fcff:fef4:85bd", start_icmp_test); +vm.on_output("Data: INCLUDEOS12345ABCDEFGHIJKLMNOPQRSTUVWXYZ12345678", data_line); # Boot the VM, taking a timeout as parameter vm.cmake().boot(50).clean() From e692d305ce4b1f5c24a863c8055a35956110f721 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 8 Aug 2018 15:04:08 +0200 Subject: [PATCH 525/723] buddy: allocator working with overbook + tests --- api/util/alloc_buddy.hpp | 37 ++++++++++++++++++++--------- test/util/unit/buddy_alloc_test.cpp | 25 ++++++++++++------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index 99558406d6..b4344a73be 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -106,7 +106,7 @@ namespace os::mem::buddy { * If the policy is overbook, we compute the next power of two above bufsize, * otherwise the next power of two below. **/ - template + template static Size_t pool_size(Size_t bufsize){ using namespace util; @@ -124,24 +124,33 @@ namespace os::mem::buddy { // If bufsize == overhead + pow2, and overhead was too small to fit alloc // Try the next power of two recursively if(unalloc < overhead) - return Alloc::pool_size(pool_size); + return Alloc::pool_size

(pool_size); auto free = bufsize - overhead; return bits::keeplast(free); } - /** Required total bufsize to manage pool_size memory **/ - template - static Size_t required_size(Size_t pool_size) { + /** + * Maximum required bufsize to manage pool_size memory, + * assuming the whole pool will be available in memory. + **/ + static Size_t max_bufsize(Size_t pool_size) { + using namespace util; + return bits::roundto(min_size, pool_size) + + overhead(pool_size); + } + + /** Minimum total bufsize to manage pool_size memory **/ + template + static Size_t min_bufsize(Size_t pool_size) { using namespace util; if constexpr (P == Policy::overbook) { return overhead(pool_size); } - return bits::roundto(min_size, pool_size) - + overhead(pool_size); + return max_bufsize(pool_size); } Index_t node_count() const noexcept { @@ -170,8 +179,14 @@ namespace os::mem::buddy { return pool_size_; } + /** Total allocatable bytes, including allocated and free */ + Size_t capacity() const noexcept { + return addr_limit_ - start_addr_; + } + + /** Bytes unavailable in an overbooking allocator */ Size_t bytes_unavailable() const noexcept { - return pool_size_ - (addr_limit_ - start_addr_); + return pool_size_ - capacity(); } Size_t bytes_used() const noexcept { @@ -239,13 +254,13 @@ namespace os::mem::buddy { template static Alloc* create(void* addr, Size_t bufsize) { using namespace util; - Size_t pool_size_ = pool_size(bufsize); - Expects(bufsize >= required_size(pool_size_)); + Size_t pool_size_ = pool_size

(bufsize); + Expects(bufsize >= min_bufsize

(pool_size_)); // Placement new an allocator on addr, passing in the rest of memory auto* alloc_begin = (char*)addr + sizeof(Alloc); auto* alloc = new (addr) Alloc(alloc_begin, - bufsize + sizeof(Alloc), + bufsize - sizeof(Alloc), pool_size_); return alloc; } diff --git a/test/util/unit/buddy_alloc_test.cpp b/test/util/unit/buddy_alloc_test.cpp index c5b7d70a3a..2de205742b 100644 --- a/test/util/unit/buddy_alloc_test.cpp +++ b/test/util/unit/buddy_alloc_test.cpp @@ -24,13 +24,14 @@ #endif struct Pool { - using Alloc = mem::buddy::Alloc; + using Alloc = os::mem::buddy::Alloc; + static constexpr auto P = Alloc::Policy::overbook; Pool(size_t s) : size{s} { - auto sz = Alloc::required_size(s); + auto sz = Alloc::max_bufsize(s); auto res = posix_memalign(&addr, Alloc::min_size, sz); Expects(res == 0); - alloc = Alloc::create(addr, sz); + alloc = Alloc::create

(addr, sz); } ~Pool() { @@ -57,9 +58,15 @@ CASE("mem::buddy init allocator"){ EXPECT(bool(alloc.root())); - EXPECT(alloc.root().height() == 1); + printf("Allocator \n"); + printf("Node count: %i \n", alloc.node_count()); + printf("Pool size : %i \n", alloc.pool_size()); + + //std::cout << alloc.summary() << "\n"; - EXPECT(alloc.root().is_parent()); + EXPECT(alloc.root().height() == 1); + EXPECT(not alloc.root().is_leaf()); + EXPECT(not alloc.root().is_leaf()); EXPECT(alloc.root().is_free()); EXPECT(alloc.root().left().height() == 2); EXPECT(alloc.bytes_used() == 0); @@ -196,8 +203,8 @@ CASE("mem::buddy random ordered allocation then deallocation"){ struct Allocation { using Alloc = Pool::Alloc; - using Addr_t = mem::buddy::Addr_t; - using Size_t = mem::buddy::Size_t; + using Addr_t = os::mem::buddy::Addr_t; + using Size_t = os::mem::buddy::Size_t; Addr_t addr = 0; Size_t size = 0; @@ -247,7 +254,7 @@ CASE("mem::buddy random chaos with data verification"){ auto& alloc = *pool.alloc; EXPECT(bool(alloc.root())); - EXPECT(alloc.bytes_free() == alloc.pool_size()); + EXPECT(alloc.bytes_free() == alloc.capacity()); std::vector allocs; @@ -350,7 +357,7 @@ CASE("mem::buddy as std::allocator") { Pool pool(1_GiB); auto* resource = pool.alloc; - std::vector> numbers(resource); + std::vector> numbers(resource); EXPECT(resource->empty()); numbers.push_back(10); From ce590d415fafefc2749632d51a8805d342cb4f1a Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 8 Aug 2018 17:10:36 +0200 Subject: [PATCH 526/723] buddy: prevent wrap-around of pool size on 32-bit --- api/util/alloc_buddy.hpp | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index b4344a73be..0d2b6fcc42 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -113,11 +113,16 @@ namespace os::mem::buddy { auto pool_size = 0; // Find closest usable power of two depending on policy if constexpr (P == Policy::overbook) { - return bits::next_pow2(bufsize); - } else { - pool_size = bits::keeplast(bufsize - 1); // -1 starts the recursion + auto pow2 = bits::next_pow2(bufsize); + + // On 32 bit we might easily overflow + if (pow2 > bufsize) + return pow2; + } + pool_size = bits::keeplast(bufsize - 1); // -1 starts the recursion + auto unalloc = bufsize - pool_size; auto overhead = Alloc::overhead(pool_size); @@ -255,8 +260,8 @@ namespace os::mem::buddy { static Alloc* create(void* addr, Size_t bufsize) { using namespace util; Size_t pool_size_ = pool_size

(bufsize); + Expects(pool_size_ >= min_size); Expects(bufsize >= min_bufsize

(pool_size_)); - // Placement new an allocator on addr, passing in the rest of memory auto* alloc_begin = (char*)addr + sizeof(Alloc); auto* alloc = new (addr) Alloc(alloc_begin, From f322f3232abdd2e9424d56a0e4883ac4018930d8 Mon Sep 17 00:00:00 2001 From: John Zhang Date: Thu, 9 Aug 2018 01:21:33 +0800 Subject: [PATCH 527/723] refactor emplace back --- api/fs/vfs.hpp | 4 ++-- lib/LiveUpdate/serialize_tcp.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/api/fs/vfs.hpp b/api/fs/vfs.hpp index 17d5caa670..8f64f5b6b3 100644 --- a/api/fs/vfs.hpp +++ b/api/fs/vfs.hpp @@ -285,13 +285,13 @@ namespace fs { }; Obs_ptr insert_parent(const std::string& token){ - children_.emplace_back(new VFS_entry{token, "Directory"}); + children_.emplace_back(std::make_unique(token, "Directory")); return children_.back().get(); } template VFS_entry& insert(const std::string& token, T& obj, const std::string& desc) { - children_.emplace_back(new VFS_entry(obj, token, desc)); + children_.emplace_back(std::make_unique(obj, token, desc)); return *children_.back(); } diff --git a/lib/LiveUpdate/serialize_tcp.cpp b/lib/LiveUpdate/serialize_tcp.cpp index 5de7c26df0..a24fc1f417 100644 --- a/lib/LiveUpdate/serialize_tcp.cpp +++ b/lib/LiveUpdate/serialize_tcp.cpp @@ -113,7 +113,7 @@ int Write_queue::deserialize_from(void* addr) len += sizeof(write_buffer); // insert shared buffer into write queue - this->q.emplace_back(new std::vector ()); + this->q.emplace_back(std::make_shared>()); // copy data auto wbuf = this->q.back(); From 7a0737c2a4fbc531c099e47d8490aea19acbdf80 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 10 Aug 2018 11:17:31 +0200 Subject: [PATCH 528/723] elf: Silence verify printf --- src/kernel/elf.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index 379ca1800f..15d799c707 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -162,8 +162,8 @@ class ElfTables } bool verify_symbols() const { - printf("ELF verification from %p to %p (Syms=%#x, Strs=%#x)\n", - symtab.base, strtab.base + strtab.size, checksum_syms, checksum_strs); + //printf("ELF verification from %p to %p (Syms=%#x, Strs=%#x)\n", + // symtab.base, strtab.base + strtab.size, checksum_syms, checksum_strs); uint32_t csum = crc32_fast(symtab.base, symtab.entries * sizeof(ElfSym)); if (csum != checksum_syms) { From 24820f58826e19eb732eeb77dae4030027c3a57b Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 10 Aug 2018 11:51:06 +0200 Subject: [PATCH 529/723] test: Revert icmp6 test changes --- test/net/integration/icmp6/test.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/test/net/integration/icmp6/test.py b/test/net/integration/icmp6/test.py index 30958c0426..27d76a06a0 100755 --- a/test/net/integration/icmp6/test.py +++ b/test/net/integration/icmp6/test.py @@ -37,25 +37,18 @@ def start_icmp_test(trigger_line): "Destination: fe80:0:0:0:e823:fcff:fef4:85bd" in output_data and \ "Type: ECHO REPLY (129)" in output_data and \ "Code: DEFAULT (0)" in output_data and \ - "Checksum: " in output_data: - #"Data: INCLUDEOS12345ABCDEFGHIJKLMNOPQRSTUVWXYZ12345678" in output_data: + "Checksum: " in output_data and \ + "Data: INCLUDEOS12345ABCDEFGHIJKLMNOPQRSTUVWXYZ12345678" in output_data: num_successes += 1 print color.INFO(""), "Ping test succeeded" else: print color.FAIL(""), "Ping test FAILED" vm.exit(1, 666) - if num_successes == 2: + if num_successes == 1: vm.exit(0, " All ICMP tests succeeded. Process returned 0 exit status") -def data_line(text): - global num_successes - num_successes += 1 - if num_successes == 2: - vm.exit(0, " All ICMP tests succeeded. Process returned 0 exit status") - vm.on_output("Service IPv4 address: 10.0.0.52, IPv6 address: fe80:0:0:0:e823:fcff:fef4:85bd", start_icmp_test); -vm.on_output("Data: INCLUDEOS12345ABCDEFGHIJKLMNOPQRSTUVWXYZ12345678", data_line); # Boot the VM, taking a timeout as parameter vm.cmake().boot(50).clean() From ac9a6a176adc8dd62afad66301f7bf6600827eb6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 13 Aug 2018 10:58:32 +0200 Subject: [PATCH 530/723] examples: Fix IRCd cmake script --- examples/IRCd/CMakeLists.txt | 6 +++--- examples/IRCd/vm.json | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/examples/IRCd/CMakeLists.txt b/examples/IRCd/CMakeLists.txt index 7281611bc4..2973cfb59a 100644 --- a/examples/IRCd/CMakeLists.txt +++ b/examples/IRCd/CMakeLists.txt @@ -33,9 +33,9 @@ set(SOURCES ircd/transform.cpp ) -# To add your own include paths: -set(LOCAL_INCLUDES "") - +# Custom version string of the IRC server +set(VERSION "v0.3 IRCd") +add_definitions(-DIRC_SERVER_VERSION="${VERSION}") option(REAL_HW "Run on real hardware" ON) diff --git a/examples/IRCd/vm.json b/examples/IRCd/vm.json index 8f31de6ba5..97b3b36478 100644 --- a/examples/IRCd/vm.json +++ b/examples/IRCd/vm.json @@ -1,5 +1,6 @@ { "image" : "IRCd", "net" : [{"device" : "e1000"}], - "mem" : 1024 + "mem" : 1024, + "uuid": "5cae2301-7467-424b-9def-e9bcbc85cf91" } From c1a3c19efa19f75db3bf97b0c936c741c25b7a2f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 13 Aug 2018 10:58:53 +0200 Subject: [PATCH 531/723] x86: Fix SMBIOS system info UUID offset --- src/platform/x86_pc/smbios.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/platform/x86_pc/smbios.cpp b/src/platform/x86_pc/smbios.cpp index 2b92c44cd2..61a471347f 100644 --- a/src/platform/x86_pc/smbios.cpp +++ b/src/platform/x86_pc/smbios.cpp @@ -125,9 +125,10 @@ namespace x86 INFO2("Manufacturer: %s", hdr->get_string(hdr->data[0])); INFO2("Product name: %s", hdr->get_string(hdr->data[1])); { + // UUID starts at offset 0x4 after header char uuid[33]; for (int i = 0; i < 16; i++) { - sprintf(&uuid[i*2], "%02hhx", hdr->data[i]); + sprintf(&uuid[i*2], "%02hhx", hdr->data[0x4 + i]); } sysinfo.uuid = std::string(uuid, 32); INFO2("System UUID: %s", sysinfo.uuid.c_str()); From e8e3af342ab49a4935ac4cb72b80d3e92e22bc2d Mon Sep 17 00:00:00 2001 From: John Zhang Date: Tue, 14 Aug 2018 18:44:16 +0800 Subject: [PATCH 532/723] fix new without delete --- linux/src/drivers/tap_driver.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/linux/src/drivers/tap_driver.cpp b/linux/src/drivers/tap_driver.cpp index edb9d86e55..f197885e29 100644 --- a/linux/src/drivers/tap_driver.cpp +++ b/linux/src/drivers/tap_driver.cpp @@ -136,6 +136,7 @@ TAP_driver::TAP_driver(const char* devname, TAP_driver::~TAP_driver() { + delete epoll_ptr; close (this->tun_fd); close (this->epoll_fd); } From 24e6fa5cf4946c38e553cb343b6a70120366f09b Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 14 Aug 2018 19:02:35 +0200 Subject: [PATCH 533/723] ip4: account for linklayer padding in small packets --- api/net/ip4/packet_ip4.hpp | 8 ++++++++ src/net/ip4/ip4.cpp | 3 +++ 2 files changed, 11 insertions(+) diff --git a/api/net/ip4/packet_ip4.hpp b/api/net/ip4/packet_ip4.hpp index 69c51914a5..e804ccf9ec 100644 --- a/api/net/ip4/packet_ip4.hpp +++ b/api/net/ip4/packet_ip4.hpp @@ -103,6 +103,14 @@ namespace net { return size() - ip_header_length(); } + /** Adjust packet size to match IP header's tot_len in case of padding */ + void adjust_size_from_header() { + auto ip_len = ip_total_length(); + if (UNLIKELY(size() > ip_len)) { + set_data_end(ip_len); + } + } + /** Get total data capacity of IP packet in bytes */ uint16_t ip_capacity() const noexcept { return capacity() - ip_header_length(); } diff --git a/src/net/ip4/ip4.cpp b/src/net/ip4/ip4.cpp index 404c5cafc5..458062241a 100644 --- a/src/net/ip4/ip4.cpp +++ b/src/net/ip4/ip4.cpp @@ -130,6 +130,9 @@ namespace net { // Stat increment packets received packets_rx_++; + // Account for possible linklayer padding + packet->adjust_size_from_header(); + packet = drop_invalid_in(std::move(packet)); if (UNLIKELY(packet == nullptr)) return; From 276bd4afde49412ad17f9437aed9322e16a76ba9 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 15 Aug 2018 10:16:46 +0200 Subject: [PATCH 534/723] test: make packet factory set ip_total_length --- test/lest_util/packet_factory.hpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/lest_util/packet_factory.hpp b/test/lest_util/packet_factory.hpp index e0ae879195..7fcf25635b 100644 --- a/test/lest_util/packet_factory.hpp +++ b/test/lest_util/packet_factory.hpp @@ -53,6 +53,7 @@ static std::unique_ptr create_ip4_packet_init(ip4::Addr src, ip4 { auto ip4 = create_ip4_packet(); ip4->init(); + ip4->set_ip_total_length(ip4->size()); ip4->set_ip_src(src); ip4->set_ip_dst(dst); return ip4; @@ -82,6 +83,7 @@ static std::unique_ptr create_tcp_packet() noexcept { auto ip4 = create_ip4_packet(); ip4->init(Protocol::TCP); + ip4->set_ip_total_length(ip4->size()); auto tcp = net::static_unique_ptr_cast (std::move(ip4)); return tcp; } @@ -90,6 +92,7 @@ static std::unique_ptr create_tcp_packet_init(Socket src, Sock { auto tcp = create_tcp_packet(); tcp->init(); + tcp->set_ip_total_length(tcp->size()); tcp->set_source(src); tcp->set_destination(dst); return tcp; From 05534c75cfda18e6e6e4ecb87a2d29c70839c5f2 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 16 Aug 2018 12:51:28 +0200 Subject: [PATCH 535/723] kernel: Add version.cpp, which is always built --- api/kernel/os.hpp | 8 ++++---- linux/userspace/CMakeLists.txt | 1 + src/CMakeLists.txt | 5 +++++ src/kernel/os.cpp | 10 +++------- src/musl/uname.cpp | 6 +++--- src/version.cpp | 21 +++++++++++++++++++++ test/CMakeLists.txt | 1 + test/kernel/unit/os_test.cpp | 7 +++++-- 8 files changed, 43 insertions(+), 16 deletions(-) create mode 100644 src/version.cpp diff --git a/api/kernel/os.hpp b/api/kernel/os.hpp index b6d8a5f259..4ecd1e48bd 100644 --- a/api/kernel/os.hpp +++ b/api/kernel/os.hpp @@ -42,13 +42,13 @@ class OS { /** * Returns the OS version string **/ - static const std::string& version() noexcept + static const char* version() noexcept { return version_str_; } /** * Returns the CPU architecture for which the OS was built **/ - static const std::string& arch() noexcept + static const char* arch() noexcept { return arch_str_; } @@ -273,8 +273,8 @@ class OS { static util::KHz cpu_khz_; static uintptr_t liveupdate_loc_; - static std::string version_str_; - static std::string arch_str_; + static const char* version_str_; + static const char* arch_str_; static uintptr_t heap_begin_; static uintptr_t heap_max_; static uintptr_t memory_end_; diff --git a/linux/userspace/CMakeLists.txt b/linux/userspace/CMakeLists.txt index 3f52c7a710..4236f02ae9 100644 --- a/linux/userspace/CMakeLists.txt +++ b/linux/userspace/CMakeLists.txt @@ -1,6 +1,7 @@ set(IOS ${CMAKE_SOURCE_DIR}/..) set(NET_SOURCES + ${IOS}/src/version.cpp ${IOS}/src/net/buffer_store.cpp ${IOS}/src/net/checksum.cpp ${IOS}/src/net/super_stack.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8f9ae4fd6b..31ea6403a0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,6 +28,7 @@ endif() set(BOTAN_MODULES "net/https/botan_server.cpp") set(OS_OBJECTS + version.cpp kernel/multiboot.cpp kernel/syscalls.cpp kernel/os.cpp kernel/cpuid.cpp kernel/block.cpp kernel/events.cpp kernel/memmap.cpp kernel/pci_manager.cpp @@ -97,3 +98,7 @@ install(DIRECTORY ${INCLUDEOS_ROOT}/src/memdisk/ DESTINATION includeos/memdisk FILES_MATCHING PATTERN "*.*") install(FILES service_name.cpp DESTINATION includeos/src) + +# Invalidate files we always want to rebuild +set(REBUILD_FILES ${INCLUDEOS_ROOT}/src/version.cpp) +add_custom_target(invalidate_files ALL COMMAND ${CMAKE_COMMAND} -E touch ${REBUILD_FILES}) diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index e8e9c9691f..5a4380a6e7 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -73,10 +73,6 @@ struct Plugin_desc { }; static Fixed_vector plugins(Fixedvector_Init::UNINIT); -// OS version -std::string OS::version_str_ = OS_VERSION; -std::string OS::arch_str_ = ARCH; - void* OS::liveupdate_storage_area() noexcept { return (void*) OS::liveupdate_loc_; @@ -162,9 +158,9 @@ void OS::post_start() PROFILE("Service::start"); // begin service start FILLINE('='); - printf(" IncludeOS %s (%s / %i-bit)\n", - version().c_str(), arch().c_str(), - static_cast(sizeof(uintptr_t)) * 8); + printf(" IncludeOS %s (%s / %u-bit)\n", + version(), arch(), + static_cast(sizeof(uintptr_t)) * 8); printf(" +--> Running [ %s ]\n", Service::name()); FILLINE('~'); diff --git a/src/musl/uname.cpp b/src/musl/uname.cpp index 73df0e6985..1c31aed3f9 100644 --- a/src/musl/uname.cpp +++ b/src/musl/uname.cpp @@ -10,11 +10,11 @@ static long sys_uname(struct utsname *buf) { strcpy(buf->nodename, "IncludeOS-node"); - strcpy(buf->release, OS::version().c_str()); + strcpy(buf->release, OS::version()); - strcpy(buf->version, OS::version().c_str()); + strcpy(buf->version, OS::version()); - strcpy(buf->machine, ARCH); + strcpy(buf->machine, OS::arch()); return 0; } diff --git a/src/version.cpp b/src/version.cpp new file mode 100644 index 0000000000..2e255cf051 --- /dev/null +++ b/src/version.cpp @@ -0,0 +1,21 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +// OS version +const char* OS::version_str_ = OS_VERSION; +const char* OS::arch_str_ = ARCH; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 4fe9f88e56..fc2d85bcfd 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -162,6 +162,7 @@ set(TEST_SOURCES ) set(OS_SOURCES + ${SRC}/version.cpp ${SRC}/arch/x86_64/paging.cpp ${SRC}/fs/disk.cpp ${SRC}/fs/dirent.cpp diff --git a/test/kernel/unit/os_test.cpp b/test/kernel/unit/os_test.cpp index f6e544d2a5..9be8abfd2e 100644 --- a/test/kernel/unit/os_test.cpp +++ b/test/kernel/unit/os_test.cpp @@ -20,8 +20,11 @@ CASE("version() returns string representation of OS version") { - EXPECT(OS::version() != ""); - EXPECT(OS::version().front() == 'v'); + EXPECT(OS::version() != nullptr); + EXPECT(std::string(OS::version()).size() > 0); + EXPECT(OS::version()[0] == 'v'); + EXPECT(OS::arch() != nullptr); + EXPECT(std::string(OS::arch()).size() > 0); } CASE("cycles_since_boot() returns clock cycles since boot") From 509e81d81570594254b7516a322ffe76d8e00ab5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 16 Aug 2018 13:07:21 +0200 Subject: [PATCH 536/723] cmake: Add OS version through configured source file --- .gitignore | 1 + CMakeLists.txt | 9 +++++---- linux/CMakeLists.txt | 2 -- linux/userspace/CMakeLists.txt | 10 ++++++++++ src/CMakeLists.txt | 11 +++++++---- src/version.cpp | 21 --------------------- src/version.cpp.in | 3 +++ test/posix/integration/utsname/service.cpp | 8 ++++---- 8 files changed, 30 insertions(+), 35 deletions(-) delete mode 100644 src/version.cpp create mode 100644 src/version.cpp.in diff --git a/.gitignore b/.gitignore index 96eadfbcc6..f5646b1074 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ .DS_Store nbproject dummy.disk +src/version.cpp !Dockerfile diff --git a/CMakeLists.txt b/CMakeLists.txt index ac21b6a095..7032628009 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,6 +36,7 @@ execute_process(COMMAND git describe --dirty WORKING_DIRECTORY ${INCLUDEOS_ROOT} OUTPUT_VARIABLE OS_VERSION) string(STRIP ${OS_VERSION} OS_VERSION) +message("OS VERSION IS ${OS_VERSION}") option(cpu_feat_vanilla "Restrict use of CPU features to vanilla" ON) if(cpu_feat_vanilla) @@ -139,12 +140,12 @@ endif() # initialize C and C++ compiler flags if (CMAKE_COMPILER_IS_GNUCC) # gcc/g++ settings - set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c ${LIBCPP_THREADING} -DOS_VERSION=\\\"${OS_VERSION}\\\"") - set(CMAKE_C_FLAGS " -MMD ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c -DOS_VERSION=\"\"${OS_VERSION}\"\"") + set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c ${LIBCPP_THREADING}") + set(CMAKE_C_FLAGS " -MMD ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c") else() # these kinda work with llvm - set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c ${LIBCPP_THREADING} -DOS_VERSION=\\\"${OS_VERSION}\\\"") - set(CMAKE_C_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c -DOS_VERSION=\"\"${OS_VERSION}\"\"") + set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c ${LIBCPP_THREADING}") + set(CMAKE_C_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c") endif() # either download or cross-compile needed libraries diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index 2b11d0efbe..608fe24bfc 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -39,8 +39,6 @@ if(CUSTOM_BOTAN) include_directories("/usr/local/botan/include/botan-2") endif() -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DOS_VERSION=\\\"${OS_VERSION}\\\"") - add_definitions(-DARCH="x86_64" -DARCH_x86_64) add_definitions(-DOS_TERMINATE_ON_CONTRACT_VIOLATION) add_definitions(-DARP_PASSTHROUGH) diff --git a/linux/userspace/CMakeLists.txt b/linux/userspace/CMakeLists.txt index 4236f02ae9..372c804210 100644 --- a/linux/userspace/CMakeLists.txt +++ b/linux/userspace/CMakeLists.txt @@ -1,5 +1,15 @@ set(IOS ${CMAKE_SOURCE_DIR}/..) +# write OS version to header file +file(WRITE ${IOS}/src/version.cpp + "#include \n + const char* OS::version_str_ = \"${OS_VERSION}\";\n + const char* OS::arch_str_ = \"${ARCH}\";\n" +) +# invalidate files we always want to rebuild +set(REBUILD_FILES ${IOS}/src/version.cpp) +add_custom_target(invalidate_files ALL COMMAND ${CMAKE_COMMAND} -E touch ${REBUILD_FILES}) + set(NET_SOURCES ${IOS}/src/version.cpp ${IOS}/src/net/buffer_store.cpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 31ea6403a0..daebdd9b66 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,13 @@ # # CMake script for the OS library # +# invalidate files we always want to rebuild +set(REBUILD_FILES ${INCLUDEOS_ROOT}/src/version.cpp) +add_custom_target(invalidate_files ALL COMMAND ${CMAKE_COMMAND} -E touch ${REBUILD_FILES}) +# write OS version to source file +configure_file("${INCLUDEOS_ROOT}/src/version.cpp.in" + "${INCLUDEOS_ROOT}/src/version.cpp") + add_definitions(-DARCH_${ARCH}) add_definitions(-DARCH="${ARCH}") if (NOT threading) @@ -98,7 +105,3 @@ install(DIRECTORY ${INCLUDEOS_ROOT}/src/memdisk/ DESTINATION includeos/memdisk FILES_MATCHING PATTERN "*.*") install(FILES service_name.cpp DESTINATION includeos/src) - -# Invalidate files we always want to rebuild -set(REBUILD_FILES ${INCLUDEOS_ROOT}/src/version.cpp) -add_custom_target(invalidate_files ALL COMMAND ${CMAKE_COMMAND} -E touch ${REBUILD_FILES}) diff --git a/src/version.cpp b/src/version.cpp deleted file mode 100644 index 2e255cf051..0000000000 --- a/src/version.cpp +++ /dev/null @@ -1,21 +0,0 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org -// -// Copyright 2015-2017 Oslo and Akershus University College of Applied Sciences -// and Alfred Bratterud -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -// OS version -const char* OS::version_str_ = OS_VERSION; -const char* OS::arch_str_ = ARCH; diff --git a/src/version.cpp.in b/src/version.cpp.in new file mode 100644 index 0000000000..9b910120f8 --- /dev/null +++ b/src/version.cpp.in @@ -0,0 +1,3 @@ +#include +const char* OS::version_str_ = "${OS_VERSION}"; +const char* OS::arch_str_ = "${ARCH}"; diff --git a/test/posix/integration/utsname/service.cpp b/test/posix/integration/utsname/service.cpp index 75a1ca807c..a7f25461d6 100644 --- a/test/posix/integration/utsname/service.cpp +++ b/test/posix/integration/utsname/service.cpp @@ -26,10 +26,10 @@ int main() "sysname is IncludeOS"); CHECKSERT(strcmp(struct_test.nodename, "IncludeOS-node") == 0, "nodename is IncludeOS-node"); - CHECKSERT(strcmp(struct_test.release, OS::version().c_str()) == 0, - "release is %s", OS::version().c_str()); - CHECKSERT(strcmp(struct_test.version, OS::version().c_str()) == 0, - "version is %s", OS::version().c_str()); + CHECKSERT(strcmp(struct_test.release, OS::version()) == 0, + "release is %s", OS::version()); + CHECKSERT(strcmp(struct_test.version, OS::version()) == 0, + "version is %s", OS::version()); CHECKSERT(strcmp(struct_test.machine, ARCH) == 0, "machine is %s", ARCH); From 6311622bd281949a99514a994c9dba150a19bcf2 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 16 Aug 2018 15:45:35 +0200 Subject: [PATCH 537/723] cmake: Simplify OS version building --- .gitignore | 1 - CMakeLists.txt | 1 - src/CMakeLists.txt | 8 +------- src/version.cpp | 4 ++++ src/version.cpp.in | 3 --- 5 files changed, 5 insertions(+), 12 deletions(-) create mode 100644 src/version.cpp delete mode 100644 src/version.cpp.in diff --git a/.gitignore b/.gitignore index f5646b1074..96eadfbcc6 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,6 @@ .DS_Store nbproject dummy.disk -src/version.cpp !Dockerfile diff --git a/CMakeLists.txt b/CMakeLists.txt index 7032628009..fa66f78874 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,6 @@ execute_process(COMMAND git describe --dirty WORKING_DIRECTORY ${INCLUDEOS_ROOT} OUTPUT_VARIABLE OS_VERSION) string(STRIP ${OS_VERSION} OS_VERSION) -message("OS VERSION IS ${OS_VERSION}") option(cpu_feat_vanilla "Restrict use of CPU features to vanilla" ON) if(cpu_feat_vanilla) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index daebdd9b66..788602526d 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,13 +1,6 @@ # # CMake script for the OS library # -# invalidate files we always want to rebuild -set(REBUILD_FILES ${INCLUDEOS_ROOT}/src/version.cpp) -add_custom_target(invalidate_files ALL COMMAND ${CMAKE_COMMAND} -E touch ${REBUILD_FILES}) -# write OS version to source file -configure_file("${INCLUDEOS_ROOT}/src/version.cpp.in" - "${INCLUDEOS_ROOT}/src/version.cpp") - add_definitions(-DARCH_${ARCH}) add_definitions(-DARCH="${ARCH}") if (NOT threading) @@ -82,6 +75,7 @@ add_dependencies(os PrecompiledLibraries botan ${OPENSSL_LIBS}) # disable sanitizers on c_abi and cxx_abi, etc. set_source_files_properties(crt/c_abi.c PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") set_source_files_properties(crt/cxx_abi.cpp PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") +set_source_files_properties(version.cpp PROPERTIES COMPILE_DEFINITIONS "OS_VERSION=\"${OS_VERSION}\"") add_subdirectory(arch/${ARCH}) add_subdirectory(platform/x86_pc) diff --git a/src/version.cpp b/src/version.cpp new file mode 100644 index 0000000000..c10a73e560 --- /dev/null +++ b/src/version.cpp @@ -0,0 +1,4 @@ +#include + +const char* OS::version_str_ = OS_VERSION; +const char* OS::arch_str_ = ARCH; diff --git a/src/version.cpp.in b/src/version.cpp.in deleted file mode 100644 index 9b910120f8..0000000000 --- a/src/version.cpp.in +++ /dev/null @@ -1,3 +0,0 @@ -#include -const char* OS::version_str_ = "${OS_VERSION}"; -const char* OS::arch_str_ = "${ARCH}"; From d0790af3fab11e7aad91ba3ae9a31984d43a665a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 17 Aug 2018 14:22:20 +0200 Subject: [PATCH 538/723] net: avoid including the actual end in SACK contains --- api/net/tcp/sack.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/net/tcp/sack.hpp b/api/net/tcp/sack.hpp index 00c50d3a92..e034a21615 100644 --- a/api/net/tcp/sack.hpp +++ b/api/net/tcp/sack.hpp @@ -63,7 +63,7 @@ struct Block { bool contains(const seq_t seq) const noexcept { return static_cast(seq - start) >= 0 and - static_cast(seq - end) <= 0; + static_cast(seq - end) < 0; } bool precedes(const seq_t seq) const noexcept From ac0991f4a75ee1b43b47e265685b02b54f432673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 17 Aug 2018 14:25:39 +0200 Subject: [PATCH 539/723] tcp: introduce feature to set buffer sizes per TCP stack --- api/net/tcp/common.hpp | 5 +++++ api/net/tcp/tcp.hpp | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/api/net/tcp/common.hpp b/api/net/tcp/common.hpp index d2b3b9837b..c7a4fd3b82 100644 --- a/api/net/tcp/common.hpp +++ b/api/net/tcp/common.hpp @@ -23,6 +23,7 @@ #include #include #include +#include namespace net { namespace tcp { @@ -48,6 +49,10 @@ namespace net { static const std::chrono::seconds default_msl {30}; static const std::chrono::milliseconds default_dack_timeout {40}; + using namespace util::literals; + static constexpr size_t default_min_bufsize {4_KiB}; + static constexpr size_t default_max_bufsize {256_KiB}; + using Address = net::Addr; /** A port */ diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index 358eaafd26..17a8626d48 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -30,6 +30,7 @@ #include // writeq #include #include +#include namespace net { @@ -369,6 +370,36 @@ namespace net { uint16_t max_syn_backlog() const { return max_syn_backlog_; } + /** + * @brief Sets the minimum buffer size. + * + * @param[in] size The size + */ + void set_min_bufsize(const size_t size) + { + Expects(util::bits::is_pow2(size)); + Expects(size <= max_bufsize_); + min_bufsize_ = size; + } + + /** + * @brief Sets the maximum buffer size. + * + * @param[in] size The size + */ + void set_max_bufsize(const size_t size) + { + Expects(util::bits::is_pow2(size)); + Expects(size >= min_bufsize_); + max_bufsize_ = size; + } + + auto min_bufsize() const + { return min_bufsize_; } + + auto max_bufsize() const + { return max_bufsize_; } + /** * @brief The Maximum Segment Size to be used for this instance. * [RFC 793] [RFC 879] [RFC 6691] @@ -517,6 +548,9 @@ namespace net { /** Maximum SYN queue backlog */ uint16_t max_syn_backlog_; + size_t min_bufsize_ {tcp::default_min_bufsize}; + size_t max_bufsize_ {tcp::default_max_bufsize}; + /** Stats */ uint64_t* bytes_rx_ = nullptr; uint64_t* bytes_tx_ = nullptr; From e28cc43335d1d85747a7859bb567abb4c39e3f38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 17 Aug 2018 14:28:33 +0200 Subject: [PATCH 540/723] tcp: read buffer is now dynamically growing from min to max --- api/net/tcp/read_buffer.hpp | 15 ++++++++++++--- api/net/tcp/read_request.hpp | 2 +- lib/LiveUpdate/serialize_tcp.cpp | 2 +- src/net/tcp/connection.cpp | 2 +- src/net/tcp/read_buffer.cpp | 19 ++++++++++++++----- src/net/tcp/read_request.cpp | 6 +++--- 6 files changed, 32 insertions(+), 14 deletions(-) diff --git a/api/net/tcp/read_buffer.hpp b/api/net/tcp/read_buffer.hpp index 453543a051..aa00d48631 100644 --- a/api/net/tcp/read_buffer.hpp +++ b/api/net/tcp/read_buffer.hpp @@ -37,10 +37,18 @@ class Read_buffer { /** * @brief Construct a read buffer. * - * @param[in] capacity The capacity of the buffer + * p@aram[in] capacity The capacity of the buffer * @param[in] seq The sequence number to start on */ - Read_buffer(const size_t capacity, const seq_t start); + /** + * @brief Construct a read buffer. + * Min and max need to be power of 2. + * + * @param[in] start The sequence number the buffer starts on + * @param[in] min The minimum size of the buffer (preallocated) + * @param[in] max The maximum size of the buffer (how much it can grow) + */ + Read_buffer(const seq_t start, const size_t min, const size_t max); /** * @brief Insert data into the buffer relative to the sequence number. @@ -93,7 +101,7 @@ class Read_buffer { * @return The capacity */ size_t capacity() const noexcept - { return buf->capacity(); } + { return cap; } /** * @brief How far into the internal buffer data has been written. @@ -168,6 +176,7 @@ class Read_buffer { private: buffer_t buf; seq_t start; + size_t cap; int32_t hole; // number of bytes missing bool push_seen{false}; diff --git a/api/net/tcp/read_request.hpp b/api/net/tcp/read_request.hpp index 1df547afcf..e5ed55c2cf 100644 --- a/api/net/tcp/read_request.hpp +++ b/api/net/tcp/read_request.hpp @@ -34,7 +34,7 @@ class Read_request { static constexpr size_t buffer_limit = 2; ReadCallback callback; - Read_request(size_t size, seq_t start, ReadCallback cb); + Read_request(seq_t start, size_t min, size_t max, ReadCallback cb); size_t insert(seq_t seq, const uint8_t* data, size_t n, bool psh = false); diff --git a/lib/LiveUpdate/serialize_tcp.cpp b/lib/LiveUpdate/serialize_tcp.cpp index a24fc1f417..f7b4e8a892 100644 --- a/lib/LiveUpdate/serialize_tcp.cpp +++ b/lib/LiveUpdate/serialize_tcp.cpp @@ -179,7 +179,7 @@ void Connection::deserialize_from(void* addr) auto* readq = (read_buffer*) &area->vla[writeq_len]; if (readq->capacity) { - read_request = std::make_unique(readq->capacity, readq->seq, nullptr); + read_request = std::make_unique(readq->seq, readq->capacity, host_.max_bufsize(), nullptr); read_request->front().deserialize_from(readq); } diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index d74a412ca9..e882ecb0b4 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -61,7 +61,7 @@ void Connection::_on_read(size_t recv_bufsz, ReadCallback cb) { if(read_request == nullptr) { - read_request = std::make_unique(recv_bufsz, seq_t(this->cb.RCV.NXT), cb); + read_request.reset(new Read_request(this->cb.RCV.NXT, host_.min_bufsize(), host_.max_bufsize(), cb)); } // read request is already set, only reset if new size. else diff --git a/src/net/tcp/read_buffer.cpp b/src/net/tcp/read_buffer.cpp index 807de66fe3..db420d368a 100644 --- a/src/net/tcp/read_buffer.cpp +++ b/src/net/tcp/read_buffer.cpp @@ -16,19 +16,24 @@ // limitations under the License. #include +#include namespace net { namespace tcp { -Read_buffer::Read_buffer(const size_t capacity, const seq_t startv) +Read_buffer::Read_buffer(const seq_t startv, const size_t min, const size_t max) : buf(tcp::construct_buffer()), - start{startv}, hole{0} + start{startv}, cap{max}, hole{0} { - buf->reserve(capacity); + Expects(util::bits::is_pow2(cap)); + Expects(util::bits::is_pow2(min)); + Expects(cap >= min); + buf->reserve(min); } size_t Read_buffer::insert(const seq_t seq, const uint8_t* data, size_t len, bool push) { + auto old_cap = buf->capacity(); assert(buf != nullptr && "Buffer seems to be stolen, make sure to renew()"); // get the relative sequence number (the diff) @@ -47,12 +52,16 @@ size_t Read_buffer::insert(const seq_t seq, const uint8_t* data, size_t len, boo if (rel == buf->size()) { buf->insert(buf->end(), data, data + len); } - else { - if (rel + len > buf->size()) buf->resize(rel + len); + else + { + if (rel + len > buf->size()) + buf->resize(rel + len); + __builtin_memcpy(buf->data() + rel, data, len); } if (push) push_seen = true; + return len; } diff --git a/src/net/tcp/read_request.cpp b/src/net/tcp/read_request.cpp index e613578957..b6550b261f 100644 --- a/src/net/tcp/read_request.cpp +++ b/src/net/tcp/read_request.cpp @@ -20,10 +20,10 @@ namespace net { namespace tcp { - Read_request::Read_request(size_t size, seq_t start, ReadCallback cb) + Read_request::Read_request(seq_t start, size_t min, size_t max, ReadCallback cb) : callback{cb} { - buffers.push_back(std::make_unique(size, start)); + buffers.push_back(std::make_unique(start, min, max)); } size_t Read_request::insert(seq_t seq, const uint8_t* data, size_t n, bool psh) @@ -125,7 +125,7 @@ namespace tcp { // we probably need to create multiple buffers, // ... or just decide we only support gaps of 1 buffer size. buffers.push_back( - std::make_unique(cur_back->capacity(), cur_back->end_seq())); + std::make_unique(cur_back->end_seq(), cur_back->capacity(), cur_back->capacity())); auto& back = buffers.back(); //printf("new buffer added start=%u end=%u, fits(%lu)=%lu\n", From 7ed92d0fce010fd27fc3e1b0e55c337bad5b83b5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 17 Aug 2018 16:13:32 +0200 Subject: [PATCH 541/723] linux: Add plugin, config and autoconf support --- cmake/linux.service.cmake | 37 +++++++++++++++++---- examples/TCP_perf/linux/CMakeLists.txt | 6 +++- examples/TCP_perf/linux/config.json | 11 ++++++ examples/TCP_perf/linux/run.sh | 2 ++ examples/TCP_perf/service.cpp | 5 +-- linux/CMakeLists.txt | 1 + linux/src/CMakeLists.txt | 1 + linux/src/config.cpp | 46 ++++++++++++++++++++++++++ linux/userspace/CMakeLists.txt | 14 ++------ src/plugins/autoconf.cpp | 2 +- 10 files changed, 102 insertions(+), 23 deletions(-) create mode 100644 examples/TCP_perf/linux/config.json create mode 100644 linux/src/config.cpp diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index 7ca3c7d09e..fe82840f1b 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -37,7 +37,8 @@ if(CUSTOM_BOTAN) include_directories("/usr/local/include/botan/botan-2") endif() -add_definitions(-DARCH="x86_64" -DARCH_x86_64) +set(ARCH "x86_64") +add_definitions("-DARCH=${ARCH}" "-DARCH_${ARCH}") add_definitions(-DOS_TERMINATE_ON_CONTRACT_VIOLATION) add_definitions(-DARP_PASSTHROUGH) add_definitions(-DNO_DEBUG) @@ -50,14 +51,40 @@ set(IOSPATH $ENV{INCLUDEOS_PREFIX}/includeos) # includes include_directories(${LOCAL_INCLUDES}) -include_directories(${IOSPATH}/x86_64/include) +include_directories(${IOSPATH}/${ARCH}/include) include_directories(${IOSPATH}/api) include_directories(${IOSPATH}/include) include_directories(${IOSPATH}/linux) include_directories(${IOSPATH}/../include) -set(LPATH $ENV{INCLUDEOS_PREFIX}/includeos/linux) +# linux executable +add_executable(service ${SOURCES} ${IOSPATH}/src/service_name.cpp) +set_target_properties(service PROPERTIES OUTPUT_NAME ${BINARY}) +set(LPATH ${IOSPATH}/linux) +set(PLUGIN_LOC "${IOSPATH}/${ARCH}/plugins") +set(DRIVER_LOC "${IOSPATH}/${ARCH}/drivers") + +# IncludeOS plugins +set(PLUGINS_LIST) +function(configure_plugin type name path) + add_library(${type}_${name} STATIC IMPORTED) + set_target_properties(${type}_${name} PROPERTIES LINKER_LANGUAGE CXX) + set_target_properties(${type}_${name} PROPERTIES IMPORTED_LOCATION ${path}) + set(PLUGINS_LIST ${PLUGINS_LIST} -Wl,--whole-archive ${type}_${name} -Wl,--no-whole-archive PARENT_SCOPE) +endfunction() +foreach(PNAME ${PLUGINS}) + set(PPATH "${PLUGIN_LOC}/lib${PNAME}.a") + message(STATUS "Enabling plugin: ${PNAME} --> ${PPATH}") + configure_plugin("plugin" ${PNAME} ${PPATH}) +endforeach() +foreach(DNAME ${DRIVERS}) + set(DPATH "${DRIVER_LOC}/lib${DNAME}.a") + message(STATUS "Enabling driver: ${DNAME} --> ${DPATH}") + configure_plugin("driver" ${DNAME} ${DPATH}) +endforeach() + +# static imported libraries add_library(linuxrt STATIC IMPORTED) set_target_properties(linuxrt PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(linuxrt PROPERTIES IMPORTED_LOCATION ${LPATH}/liblinuxrt.a) @@ -70,13 +97,11 @@ add_library(http_parser STATIC IMPORTED) set_target_properties(http_parser PROPERTIES LINKER_LANGUAGE CXX) set_target_properties(http_parser PROPERTIES IMPORTED_LOCATION ${LPATH}/libhttp_parser.a) -add_executable(service ${SOURCES} ${IOSPATH}/src/service_name.cpp) -set_target_properties(service PROPERTIES OUTPUT_NAME ${BINARY}) - if (CUSTOM_BOTAN) set(BOTAN_LIBS /usr/local/lib/libbotan-2.a) target_link_libraries(service ${BOTAN_LIBS} -ldl -pthread) endif() +target_link_libraries(service ${PLUGINS_LIST}) target_link_libraries(service includeos linuxrt includeos http_parser rt) target_link_libraries(service ${EXTRA_LIBS}) if (CUSTOM_BOTAN) diff --git a/examples/TCP_perf/linux/CMakeLists.txt b/examples/TCP_perf/linux/CMakeLists.txt index e7eafeaa43..da639a6401 100644 --- a/examples/TCP_perf/linux/CMakeLists.txt +++ b/examples/TCP_perf/linux/CMakeLists.txt @@ -12,7 +12,11 @@ set(BINARY "tcp_perf") # Source files to be linked with OS library parts to form bootable image set(SOURCES - ../service.cpp + ../service.cpp + ) + +set(PLUGINS + autoconf ) include($ENV{INCLUDEOS_PREFIX}/includeos/linux.service.cmake) diff --git a/examples/TCP_perf/linux/config.json b/examples/TCP_perf/linux/config.json new file mode 100644 index 0000000000..26564e1325 --- /dev/null +++ b/examples/TCP_perf/linux/config.json @@ -0,0 +1,11 @@ +{ + "net" : [ + { + "iface": 0, + "config": "static", + "address": "10.0.0.42", + "netmask": "255.255.255.0", + "gateway": "10.0.0.1" + } + ] +} diff --git a/examples/TCP_perf/linux/run.sh b/examples/TCP_perf/linux/run.sh index b02e68d52b..74a29c3252 100755 --- a/examples/TCP_perf/linux/run.sh +++ b/examples/TCP_perf/linux/run.sh @@ -1,8 +1,10 @@ #!/bin/bash +set -e mkdir -p build pushd build cmake .. make -j4 popd +set +e sudo mknod /dev/net/tap c 10 200 sudo ./build/tcp_perf diff --git a/examples/TCP_perf/service.cpp b/examples/TCP_perf/service.cpp index bf9ae25eb6..7ff2dba31f 100644 --- a/examples/TCP_perf/service.cpp +++ b/examples/TCP_perf/service.cpp @@ -117,13 +117,10 @@ void Service::ready() #ifdef USERSPACE_LINUX extern void create_network_device(int N, const char* route, const char* ip); create_network_device(0, "10.0.0.0/24", "10.0.0.1"); +#endif // Get the first IP stack configured from config.json auto& inet = net::Super_stack::get(0); - inet.network_config({10,0,0,42}, {255,255,255,0}, {10,0,0,1}); -#else - auto& inet = net::Super_stack::get(0); -#endif auto& tcp = inet.tcp(); tcp.set_DACK(dack); // default tcp.set_MSL(std::chrono::seconds(3)); diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index 608fe24bfc..7226c88ebb 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -40,6 +40,7 @@ if(CUSTOM_BOTAN) endif() add_definitions(-DARCH="x86_64" -DARCH_x86_64) +add_definitions(-DOS_VERSION=\"${OS_VERSION}\") add_definitions(-DOS_TERMINATE_ON_CONTRACT_VIOLATION) add_definitions(-DARP_PASSTHROUGH) add_definitions(-DNO_DEBUG) diff --git a/linux/src/CMakeLists.txt b/linux/src/CMakeLists.txt index b24bcee2d9..da9ee1d364 100644 --- a/linux/src/CMakeLists.txt +++ b/linux/src/CMakeLists.txt @@ -1,6 +1,7 @@ set(SOURCES arch.cpp + config.cpp main.cpp os.cpp profile.cpp diff --git a/linux/src/config.cpp b/linux/src/config.cpp new file mode 100644 index 0000000000..58b0b3cbe2 --- /dev/null +++ b/linux/src/config.cpp @@ -0,0 +1,46 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2017 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +static std::unique_ptr config = nullptr; +static std::unique_ptr buffer = nullptr; + +const Config& Config::get() noexcept +{ + if (config == nullptr) + { + auto* fp = fopen("config.json", "rb"); + assert(fp != nullptr && "Open config.json in source dir"); + fseek(fp, 0L, SEEK_END); + long int size = ftell(fp); + rewind(fp); + // read file into buffer + buffer.reset(new char[size+1]); + size_t res = fread(buffer.get(), size, 1, fp); + assert(res == 1); + // config needs null-termination + buffer[size] = 0; + // create config + config = std::unique_ptr (new Config(buffer.get(), buffer.get() + size)); + } + assert(config != nullptr); + return *config; +} diff --git a/linux/userspace/CMakeLists.txt b/linux/userspace/CMakeLists.txt index 372c804210..83faba69c3 100644 --- a/linux/userspace/CMakeLists.txt +++ b/linux/userspace/CMakeLists.txt @@ -1,19 +1,9 @@ set(IOS ${CMAKE_SOURCE_DIR}/..) -# write OS version to header file -file(WRITE ${IOS}/src/version.cpp - "#include \n - const char* OS::version_str_ = \"${OS_VERSION}\";\n - const char* OS::arch_str_ = \"${ARCH}\";\n" -) -# invalidate files we always want to rebuild -set(REBUILD_FILES ${IOS}/src/version.cpp) -add_custom_target(invalidate_files ALL COMMAND ${CMAKE_COMMAND} -E touch ${REBUILD_FILES}) - set(NET_SOURCES - ${IOS}/src/version.cpp ${IOS}/src/net/buffer_store.cpp ${IOS}/src/net/checksum.cpp + ${IOS}/src/net/configure.cpp ${IOS}/src/net/super_stack.cpp ${IOS}/src/net/inet.cpp @@ -80,6 +70,7 @@ if (CUSTOM_BOTAN) endif() set(OS_SOURCES + ${IOS}/src/version.cpp ${IOS}/src/fs/dirent.cpp ${IOS}/src/fs/disk.cpp ${IOS}/src/fs/fat.cpp @@ -95,6 +86,7 @@ set(OS_SOURCES ${IOS}/src/kernel/rng.cpp ${IOS}/src/kernel/timers.cpp ${IOS}/src/util/async.cpp + ${IOS}/src/util/autoconf.cpp ${IOS}/src/util/crc32.cpp ${IOS}/src/util/logger.cpp ${IOS}/src/util/sha1.cpp diff --git a/src/plugins/autoconf.cpp b/src/plugins/autoconf.cpp index f8c95ff096..267a07d016 100644 --- a/src/plugins/autoconf.cpp +++ b/src/plugins/autoconf.cpp @@ -19,6 +19,6 @@ #include __attribute__((constructor)) -void register_autoconf_plugin() { +static void register_autoconf_plugin() { OS::register_plugin(autoconf::run, "Autoconf plugin"); } From cf147440473e2aeb1fd49007db2d69129b5b5f31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 20 Aug 2018 10:54:56 +0200 Subject: [PATCH 542/723] util: rename local var to avoid shadowing issue with g++8 --- api/util/alloc_buddy.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index 0d2b6fcc42..a0423c52c1 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -110,7 +110,7 @@ namespace os::mem::buddy { static Size_t pool_size(Size_t bufsize){ using namespace util; - auto pool_size = 0; + auto pool_sz = 0; // Find closest usable power of two depending on policy if constexpr (P == Policy::overbook) { auto pow2 = bits::next_pow2(bufsize); @@ -121,15 +121,15 @@ namespace os::mem::buddy { } - pool_size = bits::keeplast(bufsize - 1); // -1 starts the recursion + pool_sz = bits::keeplast(bufsize - 1); // -1 starts the recursion - auto unalloc = bufsize - pool_size; - auto overhead = Alloc::overhead(pool_size); + auto unalloc = bufsize - pool_sz; + auto overhead = Alloc::overhead(pool_sz); // If bufsize == overhead + pow2, and overhead was too small to fit alloc // Try the next power of two recursively if(unalloc < overhead) - return Alloc::pool_size

(pool_size); + return Alloc::pool_size

(pool_sz); auto free = bufsize - overhead; return bits::keeplast(free); From 5bffec36a9c3774f8533b17b2433f40e07fcc7b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 20 Aug 2018 10:56:29 +0200 Subject: [PATCH 543/723] tcp: Changes to how reseting a read buffer works --- api/net/tcp/read_buffer.hpp | 13 ++----------- api/net/tcp/read_request.hpp | 2 +- src/net/tcp/connection.cpp | 3 ++- src/net/tcp/read_buffer.cpp | 16 ++++++++-------- src/net/tcp/read_request.cpp | 4 ++-- test/net/unit/tcp_read_buffer_test.cpp | 18 ++++++++++-------- test/net/unit/tcp_read_request_test.cpp | 2 +- 7 files changed, 26 insertions(+), 32 deletions(-) diff --git a/api/net/tcp/read_buffer.hpp b/api/net/tcp/read_buffer.hpp index aa00d48631..ac4899326b 100644 --- a/api/net/tcp/read_buffer.hpp +++ b/api/net/tcp/read_buffer.hpp @@ -34,12 +34,6 @@ namespace tcp { */ class Read_buffer { public: - /** - * @brief Construct a read buffer. - * - * p@aram[in] capacity The capacity of the buffer - * @param[in] seq The sequence number to start on - */ /** * @brief Construct a read buffer. * Min and max need to be power of 2. @@ -181,12 +175,9 @@ class Read_buffer { bool push_seen{false}; /** - * @brief Reset the buffer if either non-unique or - * a decrease of the current capacity. - * - * @param[in] capacity The capacity + * @brief Reset the buffer if non-unique */ - void reset_buffer_if_needed(const size_t capacity); + void reset_buffer_if_needed(); }; // < class Read_buffer diff --git a/api/net/tcp/read_request.hpp b/api/net/tcp/read_request.hpp index e5ed55c2cf..a7dff15388 100644 --- a/api/net/tcp/read_request.hpp +++ b/api/net/tcp/read_request.hpp @@ -44,7 +44,7 @@ class Read_request { void set_start(seq_t seq); - void reset(size_t size, const seq_t seq); + void reset(const seq_t seq); const Read_buffer& front() const { return *buffers.front(); } diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index e882ecb0b4..c27786d21b 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -59,6 +59,7 @@ Connection::~Connection() void Connection::_on_read(size_t recv_bufsz, ReadCallback cb) { + (void) recv_bufsz; if(read_request == nullptr) { read_request.reset(new Read_request(this->cb.RCV.NXT, host_.min_bufsize(), host_.max_bufsize(), cb)); @@ -69,7 +70,7 @@ void Connection::_on_read(size_t recv_bufsz, ReadCallback cb) //printf("on_read already set\n"); read_request->callback = cb; // this will flush the current data to the user (if any) - read_request->reset(recv_bufsz, seq_t(this->cb.RCV.NXT)); + read_request->reset(this->cb.RCV.NXT); // due to throwing away buffers (and all data) we also // need to clear the sack list if anything is stored here. diff --git a/src/net/tcp/read_buffer.cpp b/src/net/tcp/read_buffer.cpp index db420d368a..6f15827cc1 100644 --- a/src/net/tcp/read_buffer.cpp +++ b/src/net/tcp/read_buffer.cpp @@ -33,7 +33,6 @@ Read_buffer::Read_buffer(const seq_t startv, const size_t min, const size_t max) size_t Read_buffer::insert(const seq_t seq, const uint8_t* data, size_t len, bool push) { - auto old_cap = buf->capacity(); assert(buf != nullptr && "Buffer seems to be stolen, make sure to renew()"); // get the relative sequence number (the diff) @@ -75,29 +74,30 @@ void Read_buffer::reset(const seq_t seq, const size_t capacity) start = seq; hole = 0; push_seen = false; - reset_buffer_if_needed(capacity); + cap = capacity; + reset_buffer_if_needed(); } -void Read_buffer::reset_buffer_if_needed(const size_t capacity) +void Read_buffer::reset_buffer_if_needed() { // if the buffer isnt unique, create a new one if (buf.use_count() != 1) { buf = tcp::construct_buffer(); - buf->reserve(capacity); + buf->reserve(cap); return; } // from here on the buffer is ours only buf->clear(); const auto bufcap = buf->capacity(); - if (UNLIKELY(capacity < bufcap)) + if (UNLIKELY(cap < bufcap)) { buf->shrink_to_fit(); - buf->reserve(capacity); + buf->reserve(cap); } - else if (UNLIKELY(capacity != bufcap)) + else if (UNLIKELY(cap != bufcap)) { - buf->reserve(capacity); + buf->reserve(cap); } } diff --git a/src/net/tcp/read_request.cpp b/src/net/tcp/read_request.cpp index b6550b261f..45d828bb99 100644 --- a/src/net/tcp/read_request.cpp +++ b/src/net/tcp/read_request.cpp @@ -185,7 +185,7 @@ namespace tcp { } } - void Read_request::reset(size_t size, const seq_t seq) + void Read_request::reset(const seq_t seq) { Expects(not buffers.empty()); @@ -207,7 +207,7 @@ namespace tcp { callback(buf->buffer()); } // reset the first buffer - buf->reset(seq, size); + buf->reset(seq); // throw the others away buffers.erase(++it, buffers.end()); diff --git a/test/net/unit/tcp_read_buffer_test.cpp b/test/net/unit/tcp_read_buffer_test.cpp index d467b47e81..ddc484c597 100644 --- a/test/net/unit/tcp_read_buffer_test.cpp +++ b/test/net/unit/tcp_read_buffer_test.cpp @@ -18,14 +18,16 @@ #include #include +static const size_t MIN_BUFSZ = 128; +static const size_t MAX_BUFSZ = 128; CASE("Filling a hole") { using namespace net::tcp; const seq_t SEQ_START = 322; seq_t SEQ = SEQ_START; - const size_t BUFSZ = 128; + const size_t BUFSZ = MIN_BUFSZ; - Read_buffer buf{BUFSZ, SEQ}; + Read_buffer buf{SEQ, MIN_BUFSZ, MAX_BUFSZ}; EXPECT(buf.size() == 0); EXPECT(buf.capacity() == BUFSZ); @@ -83,9 +85,9 @@ CASE("Filling the buffer") using namespace net::tcp; const seq_t SEQ_START = 322; seq_t SEQ = SEQ_START; - const size_t BUFSZ = 128; + const size_t BUFSZ = MIN_BUFSZ; - Read_buffer buf{BUFSZ, SEQ}; + Read_buffer buf{SEQ, MIN_BUFSZ, MAX_BUFSZ}; using namespace std::string_literals; @@ -120,9 +122,9 @@ CASE("Reseting the buffer") using namespace net::tcp; const seq_t SEQ_START = 322; seq_t SEQ = SEQ_START; - const size_t BUFSZ = 128; + const size_t BUFSZ = MIN_BUFSZ; - Read_buffer buf{BUFSZ, SEQ}; + Read_buffer buf{SEQ, MIN_BUFSZ, MAX_BUFSZ}; using namespace std::string_literals; @@ -179,7 +181,7 @@ CASE("fits()") std::unique_ptr buf; - buf.reset(new Read_buffer(BUFSZ, seq)); + buf.reset(new Read_buffer(seq, BUFSZ, BUFSZ)); EXPECT(buf->fits(1000) == BUFSZ - (1000 - seq)); EXPECT(buf->fits(1200) == BUFSZ - (1200 - seq)); @@ -189,7 +191,7 @@ CASE("fits()") const uint32_t MAX_UINT = std::numeric_limits::max(); seq = MAX_UINT - 500; - buf.reset(new Read_buffer(BUFSZ, seq)); + buf.reset(new Read_buffer(seq, BUFSZ, BUFSZ)); EXPECT(buf->fits(seq) == BUFSZ); EXPECT(buf->fits(seq + 500) == BUFSZ - 500); EXPECT(buf->fits(seq + 1000) == BUFSZ - 1000); diff --git a/test/net/unit/tcp_read_request_test.cpp b/test/net/unit/tcp_read_request_test.cpp index 39c3777e59..4ff0e4f2a6 100644 --- a/test/net/unit/tcp_read_request_test.cpp +++ b/test/net/unit/tcp_read_request_test.cpp @@ -34,7 +34,7 @@ CASE("Operating with out of order data") no_reads++; }; - auto req = std::make_unique(BUFSZ, seq, read_cb); + auto req = std::make_unique(seq, BUFSZ, BUFSZ, read_cb); no_reads = 0; // Insert hole, first missing From ca9a929b4addc07b8e9adff5dfa0036fd38a7840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 20 Aug 2018 15:04:34 +0200 Subject: [PATCH 544/723] util: store port if found in URI, discard port str ptr --- api/util/uri.hpp | 8 -------- src/util/uri.cpp | 27 ++++++++++++++++----------- test/util/unit/uri_test.cpp | 6 ------ 3 files changed, 16 insertions(+), 25 deletions(-) diff --git a/api/util/uri.hpp b/api/util/uri.hpp index daebb85db8..1b0b7969f6 100644 --- a/api/util/uri.hpp +++ b/api/util/uri.hpp @@ -184,13 +184,6 @@ class URI { /// std::string host_and_port() const; - /// - /// Get the raw port number in decimal character representation. - /// - /// @return The raw port number in decimal character representation - /// - util::sview port_str() const noexcept; - /// /// Get numeric port number. /// @@ -300,7 +293,6 @@ class URI { util::sview scheme_; util::sview userinfo_; util::sview host_; - util::sview port_str_; util::sview path_; util::sview query_; util::sview fragment_; diff --git a/src/util/uri.cpp b/src/util/uri.cpp index 5d4ce3d63c..777ba3d22a 100644 --- a/src/util/uri.cpp +++ b/src/util/uri.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "../../mod/http-parser/http_parser.h" @@ -116,7 +117,6 @@ URI::URI(const URI& u) , scheme_ {updated_copy(uri_str_, u.scheme_, u.uri_str_)} , userinfo_ {updated_copy(uri_str_, u.userinfo_, u.uri_str_)} , host_ {updated_copy(uri_str_, u.host_, u.uri_str_)} - , port_str_ {updated_copy(uri_str_, u.port_str_, u.uri_str_)} , path_ {updated_copy(uri_str_, u.path_, u.uri_str_)} , query_ {updated_copy(uri_str_, u.query_, u.uri_str_)} , fragment_ {updated_copy(uri_str_, u.fragment_, u.uri_str_)} @@ -136,7 +136,6 @@ URI::URI(URI&& u) noexcept , scheme_ {u.scheme_} , userinfo_ {u.userinfo_} , host_ {u.host_} - , port_str_ {u.port_str_} , path_ {u.path_} , query_ {u.query_} , fragment_ {u.fragment_} @@ -151,7 +150,6 @@ URI& URI::operator=(const URI& u) { scheme_ = updated_copy(uri_str_, u.scheme_, u.uri_str_); userinfo_ = updated_copy(uri_str_, u.userinfo_, u.uri_str_); host_ = updated_copy(uri_str_, u.host_, u.uri_str_); - port_str_ = updated_copy(uri_str_, u.port_str_, u.uri_str_); path_ = updated_copy(uri_str_, u.path_, u.uri_str_); query_ = updated_copy(uri_str_, u.query_, u.uri_str_); fragment_ = updated_copy(uri_str_, u.fragment_, u.uri_str_); @@ -173,7 +171,6 @@ URI& URI::operator=(URI&& u) noexcept { scheme_ = u.scheme_; userinfo_ = u.userinfo_; host_ = u.host_; - port_str_ = u.port_str_; path_ = u.path_; query_ = u.query_; fragment_ = u.fragment_; @@ -216,11 +213,6 @@ std::string URI::host_and_port() const { return std::string{host_.data(), host_.length()} + ':' + std::to_string(port_); } -/////////////////////////////////////////////////////////////////////////////// -util::sview URI::port_str() const noexcept { - return port_str_; -} - /////////////////////////////////////////////////////////////////////////////// uint16_t URI::port() const noexcept { return port_; @@ -299,12 +291,25 @@ URI& URI::parse() { scheme_ = (u.field_set & (1 << UF_SCHEMA)) ? util::sview{p + u.field_data[UF_SCHEMA].off, u.field_data[UF_SCHEMA].len} : util::sview{}; userinfo_ = (u.field_set & (1 << UF_USERINFO)) ? util::sview{p + u.field_data[UF_USERINFO].off, u.field_data[UF_USERINFO].len} : util::sview{}; host_ = (u.field_set & (1 << UF_HOST)) ? util::sview{p + u.field_data[UF_HOST].off, u.field_data[UF_HOST].len} : util::sview{}; - port_str_ = (u.field_set & (1 << UF_PORT)) ? util::sview{p + u.field_data[UF_PORT].off, u.field_data[UF_PORT].len} : util::sview{}; path_ = (u.field_set & (1 << UF_PATH)) ? util::sview{p + u.field_data[UF_PATH].off, u.field_data[UF_PATH].len} : util::sview{}; query_ = (u.field_set & (1 << UF_QUERY)) ? util::sview{p + u.field_data[UF_QUERY].off, u.field_data[UF_QUERY].len} : util::sview{}; fragment_ = (u.field_set & (1 << UF_FRAGMENT)) ? util::sview{p + u.field_data[UF_FRAGMENT].off, u.field_data[UF_FRAGMENT].len} : util::sview{}; - port_ = bind_port(scheme_, u.port); + auto port_str_ = (u.field_set & (1 << UF_PORT)) ? + util::sview{p + u.field_data[UF_PORT].off, u.field_data[UF_PORT].len} : util::sview{}; + + if(not port_str_.empty()) + { + std::array buf; + std::copy(port_str_.begin(), port_str_.end(), buf.begin()); + buf[port_str_.size()] = 0; + port_ = std::atoi(buf.data()); + } + else + { + port_ = bind_port(scheme_, u.port); + } + return *this; } diff --git a/test/util/unit/uri_test.cpp b/test/util/unit/uri_test.cpp index c8563324d8..41ec9f7045 100644 --- a/test/util/unit/uri_test.cpp +++ b/test/util/unit/uri_test.cpp @@ -114,12 +114,6 @@ CASE("host_is_ip6() returns whether uri's host is an IPv6 address") EXPECT(uri.host_is_ip6() == true); } -CASE("port_str() returns uri's port as string") -{ - uri::URI uri {"http://www.vg.no:8080"}; - EXPECT(uri.port_str() == "8080"); -} - CASE("URI construction, assignment") { uri::URI uri1 {"http://www.vg.no:8080"}; From c9e967aa0311d3f522e3563b605e8e722c190f5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 22 Aug 2018 11:03:08 +0200 Subject: [PATCH 545/723] tcp: avoid shrinking buffer capacity --- src/net/tcp/read_buffer.cpp | 25 ++++++++++++++++--------- src/net/tcp/read_request.cpp | 1 - 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/src/net/tcp/read_buffer.cpp b/src/net/tcp/read_buffer.cpp index 6f15827cc1..265d18624e 100644 --- a/src/net/tcp/read_buffer.cpp +++ b/src/net/tcp/read_buffer.cpp @@ -66,7 +66,7 @@ size_t Read_buffer::insert(const seq_t seq, const uint8_t* data, size_t len, boo void Read_buffer::reset(const seq_t seq) { - this->reset(seq, buf->capacity()); + this->reset(seq, capacity()); } void Read_buffer::reset(const seq_t seq, const size_t capacity) @@ -80,24 +80,31 @@ void Read_buffer::reset(const seq_t seq, const size_t capacity) void Read_buffer::reset_buffer_if_needed() { + // current buffer cap + const auto bufcap = buf->capacity(); + + // buffer is only ours + if(buf.unique()) + { + buf->clear(); + } // if the buffer isnt unique, create a new one - if (buf.use_count() != 1) + else { buf = tcp::construct_buffer(); - buf->reserve(cap); - return; } - // from here on the buffer is ours only - buf->clear(); - const auto bufcap = buf->capacity(); + + // This case is when we need a small buffer in front of + // another buffer due to SACK if (UNLIKELY(cap < bufcap)) { buf->shrink_to_fit(); buf->reserve(cap); } - else if (UNLIKELY(cap != bufcap)) + // if not we just reserve the same capacity as we had the last time + else { - buf->reserve(cap); + buf->reserve(bufcap); } } diff --git a/src/net/tcp/read_request.cpp b/src/net/tcp/read_request.cpp index 45d828bb99..6003acfa35 100644 --- a/src/net/tcp/read_request.cpp +++ b/src/net/tcp/read_request.cpp @@ -65,7 +65,6 @@ namespace tcp { // it means the local sequence number is much farther behind // the real one seq = end_seq - rem; - buf->reset(seq); //printf("size=1, reset rem=%u start=%u end=%u\n", // rem, buf->start_seq(), buf->end_seq()); From 4ace18e5cd291a3a1dc12fee38c486a33f67a1de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 22 Aug 2018 11:32:04 +0200 Subject: [PATCH 546/723] test: Update TCP read buffer test due to new reset --- test/net/unit/tcp_read_buffer_test.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/net/unit/tcp_read_buffer_test.cpp b/test/net/unit/tcp_read_buffer_test.cpp index ddc484c597..f618288f96 100644 --- a/test/net/unit/tcp_read_buffer_test.cpp +++ b/test/net/unit/tcp_read_buffer_test.cpp @@ -163,10 +163,10 @@ CASE("Reseting the buffer") auto* data = buf.buffer()->data(); buf.reset(SEQ, BUFSZ*2); EXPECT(buf.buffer().get() == ptr); - // but not same data - EXPECT(buf.buffer()->data() != data); + // and the same data + EXPECT(buf.buffer()->data() == data); - // decreasing the cap also means new data + // decreasing the cap means new data data = buf.buffer()->data(); buf.reset(SEQ, BUFSZ/2); EXPECT(buf.buffer()->data() != data); From f671c4d0e98bf15df1aa46e4c2aa8a9af3067e5d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 22 Aug 2018 11:57:58 +0200 Subject: [PATCH 547/723] cmake: Try doing always-rebuild differently --- cmake/post.service.cmake | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index ecbdac5bca..13c6bc9ff6 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -352,8 +352,9 @@ if ("${PLATFORM}" STREQUAL "x86_solo5") endif() # Depending on the output of this command will make it always run. Like magic. -add_custom_command(OUTPUT fake_news - COMMAND cmake -E touch_nocreate alternative_facts) +add_custom_command( + OUTPUT fake_news + COMMAND cmake -E echo) # add memdisk function(add_memdisk DISK) @@ -363,7 +364,7 @@ function(add_memdisk DISK) OUTPUT memdisk.o COMMAND python ${INSTALL_LOC}/memdisk/memdisk.py --file memdisk.asm ${DISK_RELPATH} COMMAND nasm -f ${CMAKE_ASM_NASM_OBJECT_FORMAT} memdisk.asm -o memdisk.o - DEPENDS ${DISK_RELPATH} fake_news + DEPENDS ${DISK_RELPATH} ) add_library(memdisk STATIC memdisk.o) set_target_properties(memdisk PROPERTIES LINKER_LANGUAGE CXX) @@ -378,7 +379,7 @@ function(build_memdisk FOLD) COMMAND ${INSTALL_LOC}/bin/diskbuilder -o memdisk.fat ${REL_PATH} DEPENDS fake_news ) - add_custom_target(diskbuilder ALL DEPENDS memdisk.fat) + add_custom_target(diskbuilder DEPENDS memdisk.fat) add_dependencies(service diskbuilder) add_memdisk("${CMAKE_BINARY_DIR}/memdisk.fat") endfunction() From 4fae55c1021d40cff88c13e7e78ede7fe2ec997c Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 23 Aug 2018 13:04:02 +0200 Subject: [PATCH 548/723] diskbuilder: Abort when the input folder cannot be entered --- diskimagebuild/main.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/diskimagebuild/main.cpp b/diskimagebuild/main.cpp index f1688b27a8..70b8f1e169 100644 --- a/diskimagebuild/main.cpp +++ b/diskimagebuild/main.cpp @@ -76,7 +76,14 @@ int main(int argc, char** argv) } FILE* file = fopen(output_file.c_str(), "wb"); - chdir(input_folder.c_str()); + int res = chdir(input_folder.c_str()); + if (res < 0) + { + fprintf(stderr, "Disk builder failed to enter folder '%s'!\n", + input_folder.c_str()); + fprintf(stderr, "Make corrections and try again\n"); + exit(1); + } // walk filesystem subtree fsys.gather(); From a1eaf5c4876bf1fcdaae75fed292e5e32aadb8d0 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 23 Aug 2018 13:04:51 +0200 Subject: [PATCH 549/723] examples: Minor updates to IRCd/LiU examples --- examples/IRCd/ircd/restore.cpp | 2 +- examples/LiveUpdate/CMakeLists.txt | 2 +- examples/LiveUpdate/ircd.motd | 3 --- examples/LiveUpdate/update.sh | 7 ++++--- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/examples/IRCd/ircd/restore.cpp b/examples/IRCd/ircd/restore.cpp index cd261d50ed..52bfdcfc7a 100644 --- a/examples/IRCd/ircd/restore.cpp +++ b/examples/IRCd/ircd/restore.cpp @@ -92,7 +92,7 @@ void IrcServer::deserialize(Restore& thing) if (buf.size() >= sizeof(statcounters)) { memcpy(statcounters, buf.data(), buf.size()); } - printf("* Resumed server, next id: %u\n", thing.get_id()); + //printf("* Resumed server, next id: %u\n", thing.get_id()); break; } case 20: /// channels diff --git a/examples/LiveUpdate/CMakeLists.txt b/examples/LiveUpdate/CMakeLists.txt index 53e3e7c866..a8bfb14f1a 100644 --- a/examples/LiveUpdate/CMakeLists.txt +++ b/examples/LiveUpdate/CMakeLists.txt @@ -59,7 +59,7 @@ set(LIBRARIES # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) -diskbuilder(drive) +#diskbuilder(drive) add_custom_command( OUTPUT motd.h diff --git a/examples/LiveUpdate/ircd.motd b/examples/LiveUpdate/ircd.motd index 43f146eb1a..bb09ecda76 100644 --- a/examples/LiveUpdate/ircd.motd +++ b/examples/LiveUpdate/ircd.motd @@ -7,6 +7,3 @@ ,| |_.'| |\ | || |`-'|(| '---.'| | | `-' /| | / : | .--' \ | | | |.-._) \ (_| | | | \ |(_' '--'\ | |(' '-'(_.-' | '--' / | `---. `' '-' '\ / `--' `--' `--' `-----' `------' `-----' `-------' `------' `-----' `-----' - -Test2 -Test3 diff --git a/examples/LiveUpdate/update.sh b/examples/LiveUpdate/update.sh index bdb99a1625..a58c80b59d 100755 --- a/examples/LiveUpdate/update.sh +++ b/examples/LiveUpdate/update.sh @@ -1,7 +1,8 @@ #!/bin/bash -mkdir -p build -pushd build +BFOLD=build +mkdir -p $BFOLD +pushd $BFOLD cmake .. make -j8 popd -dd if=build/liveupdate > /dev/tcp/10.0.0.42/666 +dd if=$BFOLD/liveupdate > /dev/tcp/10.0.0.42/666 From 24e141d37b76fced94b5f0bac4f154baf518937f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 23 Aug 2018 13:34:14 +0200 Subject: [PATCH 550/723] musl: Use proper stubtrace function on stubbed syscalls --- src/musl/_lseek.cpp | 10 +++++++--- src/musl/mincore.cpp | 13 +++++++++---- src/musl/munmap.cpp | 15 +++++++++++---- src/musl/openat.cpp | 4 ++-- src/musl/pipe.cpp | 18 +++++++++++++----- src/musl/poll.cpp | 12 ++++++++---- src/musl/prlimit64.cpp | 10 +++++++--- src/musl/sched_getaffinity.cpp | 10 +++++++--- src/musl/sched_yield.cpp | 10 +++++++--- src/musl/set_robust_list.cpp | 10 +++++++--- src/musl/sync.cpp | 16 +++++++++++----- src/musl/sysinfo.cpp | 10 +++++++--- 12 files changed, 96 insertions(+), 42 deletions(-) diff --git a/src/musl/_lseek.cpp b/src/musl/_lseek.cpp index 362b3326e4..b04f8fc794 100644 --- a/src/musl/_lseek.cpp +++ b/src/musl/_lseek.cpp @@ -1,7 +1,11 @@ -#include "common.hpp" +#include "stub.hpp" + +static long sys__lseek() +{ + return 0; +} extern "C" long syscall_SYS__lseek() { - STUB("_lseek"); - return 0; + return stubtrace(sys__lseek, "_lseek"); } diff --git a/src/musl/mincore.cpp b/src/musl/mincore.cpp index d0cd2d7d2e..7d966487e3 100644 --- a/src/musl/mincore.cpp +++ b/src/musl/mincore.cpp @@ -1,7 +1,12 @@ -#include "common.hpp" +#include "stub.hpp" -extern "C" -long syscall_SYS_mincore() { - STUB("mincore"); +static long sys_mincore() +{ return 0; } + +extern "C" +long syscall_SYS_mincore() +{ + return stubtrace(sys_mincore, "mincore"); +} diff --git a/src/musl/munmap.cpp b/src/musl/munmap.cpp index 5db7ca4bb5..7af8ef3814 100644 --- a/src/musl/munmap.cpp +++ b/src/musl/munmap.cpp @@ -1,11 +1,18 @@ -#include "common.hpp" +#include "stub.hpp" extern "C" void kfree(void* addr, size_t length); -extern "C" -int syscall_SYS_munmap(void *addr, size_t length) +static long sys_munmap(void *addr, size_t length) { - STUB("munmap"); + if(UNLIKELY(length == 0)) + return -EINVAL; + kfree(addr, length); return 0; } + +extern "C" +long syscall_SYS_munmap(void *addr, size_t length) +{ + return stubtrace(sys_munmap, "munmap", addr, length); +} diff --git a/src/musl/openat.cpp b/src/musl/openat.cpp index 3707331f6f..bdddf3e08b 100644 --- a/src/musl/openat.cpp +++ b/src/musl/openat.cpp @@ -2,12 +2,12 @@ #include #include -static int sys_openat(int /*dirfd*/, const char* /*path*/, int /*flags*/, mode_t) +static long sys_openat(int /*dirfd*/, const char* /*path*/, int /*flags*/, mode_t) { return -ENOSYS; } extern "C" -int syscall_SYS_openat(int dirfd, const char *pathname, int flags, mode_t mode) { +long syscall_SYS_openat(int dirfd, const char *pathname, int flags, mode_t mode) { return strace(sys_openat, "openat", dirfd, pathname, flags, mode); } diff --git a/src/musl/pipe.cpp b/src/musl/pipe.cpp index f0668a02fa..ad2635c209 100644 --- a/src/musl/pipe.cpp +++ b/src/musl/pipe.cpp @@ -1,13 +1,21 @@ -#include "common.hpp" +#include "stub.hpp" + +static long sys_pipe() +{ + return 0; +} + +static long sys_pipe2() +{ + return 0; +} extern "C" { long syscall_SYS_pipe() { - STUB("pipe"); - return 0; + return stubtrace(sys_pipe, "pipe"); } long syscall_SYS_pipe2() { - STUB("pipe2"); - return 0; + return stubtrace(sys_pipe2, "pipe2"); } } // extern "C" diff --git a/src/musl/poll.cpp b/src/musl/poll.cpp index 25070a356f..6ea09188ed 100644 --- a/src/musl/poll.cpp +++ b/src/musl/poll.cpp @@ -1,13 +1,17 @@ -#include "common.hpp" +#include "stub.hpp" #include -extern "C" -long syscall_SYS_poll(struct pollfd *fds, nfds_t nfds, int /*timeout*/) +static long sys_poll(struct pollfd *fds, nfds_t nfds, int /*timeout*/) { - STUB("poll"); for (nfds_t i = 0; i < nfds; i++) { fds[i].revents = fds[i].events; } return nfds; } + +extern "C" +long syscall_SYS_poll(struct pollfd *fds, nfds_t nfds, int timeout) +{ + return stubtrace(sys_poll, "poll", fds, nfds, timeout); +} diff --git a/src/musl/prlimit64.cpp b/src/musl/prlimit64.cpp index eaaf85582b..0528e372d7 100644 --- a/src/musl/prlimit64.cpp +++ b/src/musl/prlimit64.cpp @@ -1,7 +1,11 @@ -#include "common.hpp" +#include "stub.hpp" + +static long sys_prlimit64() +{ + return 0; +} extern "C" long syscall_SYS_prlimit64() { - STUB("prlimit64"); - return 0; + return stubtrace(sys_prlimit64, "prlimit64"); } diff --git a/src/musl/sched_getaffinity.cpp b/src/musl/sched_getaffinity.cpp index c5b13e214a..e2a8e4d46b 100644 --- a/src/musl/sched_getaffinity.cpp +++ b/src/musl/sched_getaffinity.cpp @@ -1,7 +1,11 @@ -#include "common.hpp" +#include "stub.hpp" + +static long sys_sched_getaffinity() +{ + return 0; +} extern "C" long syscall_SYS_sched_getaffinity() { - STUB("sched_getaffinity"); - return 0; + return stubtrace(sys_sched_getaffinity, "sched_getaffinity"); } diff --git a/src/musl/sched_yield.cpp b/src/musl/sched_yield.cpp index 91d3bd0af3..3cf320312c 100644 --- a/src/musl/sched_yield.cpp +++ b/src/musl/sched_yield.cpp @@ -1,7 +1,11 @@ -#include "common.hpp" +#include "stub.hpp" + +static long sys_sched_yield() +{ + return 0; +} extern "C" long syscall_SYS_sched_yield() { - STUB("sched_yield"); - return 0; + return stubtrace(sys_sched_yield, "sched_yield"); } diff --git a/src/musl/set_robust_list.cpp b/src/musl/set_robust_list.cpp index d67b3a8580..a1e180c590 100644 --- a/src/musl/set_robust_list.cpp +++ b/src/musl/set_robust_list.cpp @@ -1,7 +1,11 @@ -#include "common.hpp" +#include "stub.hpp" + +static long sys_set_robust_list() +{ + return 0; +} extern "C" long syscall_SYS_set_robust_list() { - STUB("set_robust_list"); - return 0; + return stubtrace(sys_set_robust_list, "set_robust_list"); } diff --git a/src/musl/sync.cpp b/src/musl/sync.cpp index a51f892e75..6b24a7c09d 100644 --- a/src/musl/sync.cpp +++ b/src/musl/sync.cpp @@ -1,13 +1,19 @@ -#include "common.hpp" +#include "stub.hpp" + +static long sys_sync() { + return 0; +} + +static long sys_syncfs() { + return 0; +} extern "C" { long syscall_SYS_sync() { - STUB("sync"); - return 0; + return stubtrace(sys_sync, "sync"); } long syscall_SYS_syncfs() { - STUB("syncfs"); - return 0; + return stubtrace(sys_syncfs, "syncfs"); } } diff --git a/src/musl/sysinfo.cpp b/src/musl/sysinfo.cpp index d644b84d64..06c5024cd0 100644 --- a/src/musl/sysinfo.cpp +++ b/src/musl/sysinfo.cpp @@ -1,7 +1,11 @@ -#include "common.hpp" +#include "stub.hpp" + +static long sys_sysinfo() +{ + return 0; +} extern "C" long syscall_SYS_sysinfo() { - STUB("sysinfo"); - return 0; + return stubtrace(sys_sysinfo, "sysinfo"); } From c5460522607f4c222c22e338c78208be24d3089e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 23 Aug 2018 14:09:53 +0200 Subject: [PATCH 551/723] musl: Only print stubbed syscalls if strace is enabled --- src/musl/stub.hpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/musl/stub.hpp b/src/musl/stub.hpp index d1ecaae5e3..473c578779 100644 --- a/src/musl/stub.hpp +++ b/src/musl/stub.hpp @@ -10,10 +10,13 @@ inline void stubtrace_print(const char* name, R ret, Args&&... args) { } // Strace for stubbed syscalls. -// calling the syscall, recording return value and always printing +// calling the syscall, recording return value and only printing when strace is on template -inline auto stubtrace(Fn func, const char* name, Args&&... args) { +inline auto stubtrace(Fn func, const char* name[[maybe_unused]], Args&&... args) { auto ret = func(args...); - stubtrace_print(name, ret, args...); + + if constexpr (__strace) + stubtrace_print(name, ret, args...); + return ret; } From 78714f32727f52c54ca536b6f7deb712adbc0204 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 23 Aug 2018 14:31:15 +0200 Subject: [PATCH 552/723] os: Simple timestamps to stdout output --- src/kernel/os.cpp | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 5a4380a6e7..5fb4833ebf 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -176,6 +176,22 @@ bool os_enable_boot_logging = false; __attribute__((weak)) bool os_default_stdout = false; +#include +static std::string now() +{ + auto tnow = time(0); + auto* curtime = localtime(&tnow); + + char buff[48]; + int len = strftime(buff, sizeof(buff), "%T", curtime); + return std::string(buff, len); +} +bool contains(const char* str, size_t len, char c) +{ + for (size_t i = 0; i < len; i++) if (str[i] == c) return true; + return false; +} + void OS::print(const char* str, const size_t len) { if (UNLIKELY(! __libc_initialized)) { @@ -185,6 +201,19 @@ void OS::print(const char* str, const size_t len) for (auto& callback : os_print_handlers) { if (os_enable_boot_logging || OS::is_booted() || OS::is_panicking()) + { + /** TIMESTAMPING **/ + static bool ts_shown = false; + if (ts_shown == false) + { + std::string ts = "[" + now() + "] "; + callback(ts.c_str(), ts.size()); + ts_shown = true; + } + const bool has_newline = contains(str, len, '\n'); + if (ts_shown && has_newline) ts_shown = false; + /** TIMESTAMPING **/ callback(str, len); + } } } From 33a06ca5afbd1864f3c8b098689587561643129f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 23 Aug 2018 15:44:12 +0200 Subject: [PATCH 553/723] os: Add timestamps driver that adds timestamps to stdout --- api/kernel/os.hpp | 7 +++++++ src/drivers/CMakeLists.txt | 4 ++++ src/drivers/timestamps.cpp | 24 ++++++++++++++++++++++++ src/kernel/os.cpp | 38 +++++++++++++++++++------------------- 4 files changed, 54 insertions(+), 19 deletions(-) create mode 100644 src/drivers/timestamps.cpp diff --git a/api/kernel/os.hpp b/api/kernel/os.hpp index 4ecd1e48bd..4360fa6c60 100644 --- a/api/kernel/os.hpp +++ b/api/kernel/os.hpp @@ -152,6 +152,12 @@ class OS { */ static void print(const char* ptr, const size_t len); + /** + * Enable or disable timestamps automatically + * prepended to all OS::print(...) calls + */ + static void enable_timestamps(bool enabled); + /** * Add handler for standard output. */ @@ -270,6 +276,7 @@ class OS { static bool boot_sequence_passed_; static bool m_is_live_updated; static bool m_block_drivers_ready; + static bool m_timestamps; static util::KHz cpu_khz_; static uintptr_t liveupdate_loc_; diff --git a/src/drivers/CMakeLists.txt b/src/drivers/CMakeLists.txt index 8352e7c846..1a80731c47 100644 --- a/src/drivers/CMakeLists.txt +++ b/src/drivers/CMakeLists.txt @@ -49,6 +49,9 @@ add_dependencies(disk_logger PrecompiledLibraries) add_library(disklog_reader STATIC "disklog_reader.cpp") add_dependencies(disklog_reader PrecompiledLibraries) +add_library(timestamps STATIC timestamps.cpp) +add_dependencies(timestamps PrecompiledLibraries) + add_library(vga_output STATIC vgaout.cpp) add_dependencies(vga_output PrecompiledLibraries) @@ -68,6 +71,7 @@ install(TARGETS ip4_reassembly heap_debugging boot_logger disk_logger disklog_reader + timestamps vga_output vga_emergency DESTINATION includeos/${ARCH}/drivers) diff --git a/src/drivers/timestamps.cpp b/src/drivers/timestamps.cpp new file mode 100644 index 0000000000..b1687fe8ea --- /dev/null +++ b/src/drivers/timestamps.cpp @@ -0,0 +1,24 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +__attribute__((constructor)) +static void enable_timestamps() +{ + OS::enable_timestamps(true); +} diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 5fb4833ebf..d23932d360 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -54,6 +54,7 @@ bool OS::power_ = true; bool OS::boot_sequence_passed_ = false; bool OS::m_is_live_updated = false; bool OS::m_block_drivers_ready = false; +bool OS::m_timestamps = false; KHz OS::cpu_khz_ {-1}; uintptr_t OS::liveupdate_loc_ = 0; @@ -176,16 +177,7 @@ bool os_enable_boot_logging = false; __attribute__((weak)) bool os_default_stdout = false; -#include -static std::string now() -{ - auto tnow = time(0); - auto* curtime = localtime(&tnow); - - char buff[48]; - int len = strftime(buff, sizeof(buff), "%T", curtime); - return std::string(buff, len); -} +#include bool contains(const char* str, size_t len, char c) { for (size_t i = 0; i < len; i++) if (str[i] == c) return true; @@ -202,18 +194,26 @@ void OS::print(const char* str, const size_t len) for (auto& callback : os_print_handlers) { if (os_enable_boot_logging || OS::is_booted() || OS::is_panicking()) { - /** TIMESTAMPING **/ - static bool ts_shown = false; - if (ts_shown == false) + if (OS::m_timestamps && OS::is_booted() && !OS::is_panicking()) { - std::string ts = "[" + now() + "] "; - callback(ts.c_str(), ts.size()); - ts_shown = true; + /** TIMESTAMPING **/ + static bool ts_shown = false; + if (ts_shown == false) + { + std::string ts = "[" + isotime::now() + "] "; + callback(ts.c_str(), ts.size()); + ts_shown = true; + } + const bool has_newline = contains(str, len, '\n'); + if (ts_shown && has_newline) ts_shown = false; + /** TIMESTAMPING **/ } - const bool has_newline = contains(str, len, '\n'); - if (ts_shown && has_newline) ts_shown = false; - /** TIMESTAMPING **/ callback(str, len); } } } + +void OS::enable_timestamps(const bool enabled) +{ + OS::m_timestamps = enabled; +} From 7765032d7003c06d89fda8b388b603ee1ab815cc Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 24 Aug 2018 09:17:13 +0200 Subject: [PATCH 554/723] pci: Make room for 64 PCI devices of each type --- src/kernel/pci_manager.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kernel/pci_manager.cpp b/src/kernel/pci_manager.cpp index c7e538d1de..3042bf1853 100644 --- a/src/kernel/pci_manager.cpp +++ b/src/kernel/pci_manager.cpp @@ -24,7 +24,7 @@ #include #include -static const int ELEMENTS = 16; +static const int ELEMENTS = 64; template using Driver_entry = std::pair; From c440316491d50d07f8d3a4805ccef1cce26575a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 27 Aug 2018 16:04:50 +0200 Subject: [PATCH 555/723] musl: strace now prints error, replace as many stubs with enosys as possible --- src/musl/_lseek.cpp | 2 +- src/musl/common.hpp | 13 ++++-- src/musl/dup3.cpp | 8 ++-- src/musl/execve.cpp | 4 +- src/musl/fchmod.cpp | 4 +- src/musl/fchmodat.cpp | 4 +- src/musl/getrlimit.cpp | 6 +-- src/musl/ioctl.cpp | 10 ++--- src/musl/lseek.cpp | 5 +-- src/musl/mincore.cpp | 12 +++--- src/musl/mmap.diff | 79 ---------------------------------- src/musl/mremap.cpp | 13 ++++-- src/musl/msync.cpp | 11 +++-- src/musl/munmap.cpp | 4 +- src/musl/nanosleep.cpp | 12 ++++-- src/musl/pipe.cpp | 18 ++++---- src/musl/prlimit64.cpp | 16 ++++--- src/musl/rename.cpp | 7 +-- src/musl/sched_getaffinity.cpp | 14 +++--- src/musl/set_robust_list.cpp | 10 ++--- src/musl/sysinfo.cpp | 11 ++--- 21 files changed, 110 insertions(+), 153 deletions(-) delete mode 100644 src/musl/mmap.diff diff --git a/src/musl/_lseek.cpp b/src/musl/_lseek.cpp index b04f8fc794..8a6e0800ce 100644 --- a/src/musl/_lseek.cpp +++ b/src/musl/_lseek.cpp @@ -2,7 +2,7 @@ static long sys__lseek() { - return 0; + return -ENOSYS; } extern "C" diff --git a/src/musl/common.hpp b/src/musl/common.hpp index f6e2d78509..62be9175d7 100644 --- a/src/musl/common.hpp +++ b/src/musl/common.hpp @@ -57,8 +57,14 @@ inline void strace_print(const char* name, Ret ret, Args&&... args){ out << name << "("; pr_param(out, args...); out << ") = " << ret; - //if (errno) - // out << " " << strerror(errno); + + // print error string if syscall returns a negative error code + if constexpr(std::is_integral_v) { + if (static_cast(ret) > -4096UL) { + out << " " << strerror(errno); + } + } + out << '\n'; auto str = out.str(); __serial_print(str.data(), str.size()); @@ -66,12 +72,11 @@ inline void strace_print(const char* name, Ret ret, Args&&... args){ // strace, calling the syscall, recording return value and printing if enabled template -inline auto strace(Fn func, const char* name, Args&&... args) { +inline auto strace(Fn func, [[maybe_unused]]const char* name, Args&&... args) { auto ret = func(args...); if constexpr (__strace) strace_print(name, ret, args...); - (void) name; return ret; } diff --git a/src/musl/dup3.cpp b/src/musl/dup3.cpp index cfcb1ac81f..b90c8ac9dd 100644 --- a/src/musl/dup3.cpp +++ b/src/musl/dup3.cpp @@ -1,4 +1,4 @@ -#include "stub.hpp" +#include "common.hpp" #include #include @@ -17,15 +17,15 @@ static long sys_dup3(int /*oldfd*/, int /*newfd*/, int /*flags*/) extern "C" long syscall_SYS_dup(int oldfd) { - return stubtrace(sys_dup, "dup", oldfd); + return strace(sys_dup, "dup", oldfd); } extern "C" long syscall_SYS_dup2(int oldfd, int newfd) { - return stubtrace(sys_dup2, "dup2", oldfd, newfd); + return strace(sys_dup2, "dup2", oldfd, newfd); } extern "C" long syscall_SYS_dup3(int oldfd, int newfd, int flags) { - return stubtrace(sys_dup3, "dup3", oldfd, newfd, flags); + return strace(sys_dup3, "dup3", oldfd, newfd, flags); } diff --git a/src/musl/execve.cpp b/src/musl/execve.cpp index 6d2e231b0c..36c8afe7a8 100644 --- a/src/musl/execve.cpp +++ b/src/musl/execve.cpp @@ -1,4 +1,4 @@ -#include "stub.hpp" +#include "common.hpp" #include #include @@ -11,5 +11,5 @@ extern "C" long syscall_SYS_execve(const char *filename, char *const argv[], char *const envp[]) { - return stubtrace(sys_execve, "execve", filename, argv, envp); + return strace(sys_execve, "execve", filename, argv, envp); } diff --git a/src/musl/fchmod.cpp b/src/musl/fchmod.cpp index 0d2d31eab6..3faac47510 100644 --- a/src/musl/fchmod.cpp +++ b/src/musl/fchmod.cpp @@ -1,12 +1,12 @@ #include "common.hpp" #include -static int sys_fchmod(int /*fd*/, mode_t /*mode*/) { +static long sys_fchmod(int /*fd*/, mode_t /*mode*/) { // currently makes no sense, especially since we're read-only return -EROFS; } extern "C" -int syscall_SYS_fchmod(int fildes, mode_t mode) { +long syscall_SYS_fchmod(int fildes, mode_t mode) { return strace(sys_fchmod, "fchmod", fildes, mode); } diff --git a/src/musl/fchmodat.cpp b/src/musl/fchmodat.cpp index 1029550e68..7568e498d4 100644 --- a/src/musl/fchmodat.cpp +++ b/src/musl/fchmodat.cpp @@ -2,13 +2,13 @@ #include #include -static int sys_fchmodat(int /*fd*/, const char* /*path*/, mode_t, int /*flag*/) +static long sys_fchmodat(int /*fd*/, const char* /*path*/, mode_t, int /*flag*/) { // currently makes no sense, especially since we're read-only return -EROFS; } extern "C" -int syscall_SYS_fchmodat(int fd, const char *path, mode_t mode, int flag) { +long syscall_SYS_fchmodat(int fd, const char *path, mode_t mode, int flag) { return strace(sys_fchmodat, "fchmodat", fd, path, mode, flag); } diff --git a/src/musl/getrlimit.cpp b/src/musl/getrlimit.cpp index 7db87424f9..6376691621 100644 --- a/src/musl/getrlimit.cpp +++ b/src/musl/getrlimit.cpp @@ -1,6 +1,6 @@ -#include +#include "common.hpp" -#include "stub.hpp" +#include static int sys_getrlimit(int /*resource*/, struct rlimit*) { return -ENOSYS; @@ -8,5 +8,5 @@ static int sys_getrlimit(int /*resource*/, struct rlimit*) { extern "C" long syscall_SYS_getrlimit(int resource, struct rlimit *rlim) { - return stubtrace(sys_getrlimit, "getrlimit", resource, rlim); + return strace(sys_getrlimit, "getrlimit", resource, rlim); } diff --git a/src/musl/ioctl.cpp b/src/musl/ioctl.cpp index 5146cbf7f1..31ff547f73 100644 --- a/src/musl/ioctl.cpp +++ b/src/musl/ioctl.cpp @@ -1,9 +1,9 @@ +#include "common.hpp" + #include #include -#include "stub.hpp" - -static int sys_ioctl(int fd, int req, void* arg) { +static long sys_ioctl(int fd, int req, void* arg) { if (fd == 1 or fd == 2) { if (req == TIOCGWINSZ) { winsize* ws = (winsize*)arg; @@ -27,6 +27,6 @@ static int sys_ioctl(int fd, int req, void* arg) { } extern "C" -int syscall_SYS_ioctl(int fd, int req, void* arg) { - return stubtrace(sys_ioctl, "ioctl", fd, req, arg); +long syscall_SYS_ioctl(int fd, int req, void* arg) { + return strace(sys_ioctl, "ioctl", fd, req, arg); } diff --git a/src/musl/lseek.cpp b/src/musl/lseek.cpp index 4d1cc02858..86716f01d7 100644 --- a/src/musl/lseek.cpp +++ b/src/musl/lseek.cpp @@ -3,7 +3,6 @@ #include #include -#include "stub.hpp" #include static off_t sys_lseek(int fd, off_t offset, int whence) @@ -30,6 +29,6 @@ extern "C" off_t syscall_SYS__llseek(unsigned int fd, unsigned long offset_high, unsigned long offset_low, loff_t *result, unsigned int whence) { - return stubtrace(sys__llseek, "_llseek", fd, offset_high, offset_low, - result, whence); + return strace(sys__llseek, "_llseek", fd, offset_high, offset_low, + result, whence); } diff --git a/src/musl/mincore.cpp b/src/musl/mincore.cpp index 7d966487e3..ed040148df 100644 --- a/src/musl/mincore.cpp +++ b/src/musl/mincore.cpp @@ -1,12 +1,14 @@ -#include "stub.hpp" +#include "common.hpp" -static long sys_mincore() +static long sys_mincore([[maybe_unused]]void *addr, + [[maybe_unused]]size_t length, + [[maybe_unused]]unsigned char *vec) { - return 0; + return -ENOSYS; } extern "C" -long syscall_SYS_mincore() +long syscall_SYS_mincore(void *addr, size_t length, unsigned char *vec) { - return stubtrace(sys_mincore, "mincore"); + return strace(sys_mincore, "mincore", addr, length, vec); } diff --git a/src/musl/mmap.diff b/src/musl/mmap.diff deleted file mode 100644 index 558adef373..0000000000 --- a/src/musl/mmap.diff +++ /dev/null @@ -1,79 +0,0 @@ -diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp -index 511b92a..a6c836c 100644 ---- a/src/musl/mmap.cpp -+++ b/src/musl/mmap.cpp -@@ -1,19 +1,71 @@ - #include "common.hpp" - #include - +#include - +#include - +#include - +#include - - extern uintptr_t heap_begin; - +uintptr_t heap_end = 0; - static uintptr_t current_pos = 0; - + - +using Alloc = util::alloc::Lstack<4096>; - +static Alloc alloc; - + - +void init_mmap(uintptr_t addr_begin){ - + if (alloc.empty()){ - + printf("Alloc not empty: %s\n ", alloc.begin() == nullptr ? "nullptr" : "size not 0"); - + } - + Expects(alloc.empty()); - + auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); - + alloc.donate((void*)aligned_begin, (OS::heap_max() - aligned_begin) & ~(Alloc::align - 1)); - + - +} - + - +void* kalloc(size_t size){ - + return alloc.allocate(size); - +} - + - +void kfree (void* ptr, size_t size){ - + alloc.deallocate(ptr, size); - +} - + - extern "C" - void* syscall_SYS_mmap(void *addr, size_t length, int prot, int flags, - int fd, off_t offset) - { - - uintptr_t res = heap_begin + current_pos; - - current_pos += length; - + - + STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d \n", - + addr, length, prot, flags, fd, offset); - + - + // TODO: Mapping to file descriptor - + if (fd > 0) { - + assert(false && "Mapping to file descriptor not yet implemented"); - + } - + - + // TODO: mapping virtual address - + if (addr) { - + errno = ENODEV; - + return MAP_FAILED; - + } - + - + void* res = kalloc(length); - STRACE("syscall mmap: addr=%p len=%u prot=%d fl=%d fd=%d off=%d res=%p\n", - addr, length, prot, flags, fd, offset, res); - - return (void*)res; - + - + return res; - } - -+ -+/** -+ The mmap2() system call provides the same interface as mmap(2), -+ except that the final argument specifies the offset into the file in -+ 4096-byte units (instead of bytes, as is done by mmap(2)). This -+ enables applications that use a 32-bit off_t to map large files (up -+ to 2^44 bytes). -+ -+ http://man7.org/linux/man-pages/man2/mmap2.2.html -+**/ - extern "C" - void* syscall_SYS_mmap2(void *addr, size_t length, int prot, - int flags, int fd, off_t offset) { diff --git a/src/musl/mremap.cpp b/src/musl/mremap.cpp index 3941db8f1a..3fdc09784f 100644 --- a/src/musl/mremap.cpp +++ b/src/musl/mremap.cpp @@ -1,7 +1,14 @@ #include "common.hpp" -extern "C" -long syscall_SYS_mremap() { - STUB("mremap"); +static long sys_mremap(void */*old_address*/, size_t /*old_size*/, + size_t /*new_size*/, int /*flags*/, void */*new_address*/) +{ return -ENOSYS; } + +extern "C" +long syscall_SYS_mremap(void *old_address, size_t old_size, + size_t new_size, int flags, void *new_address) +{ + return strace(sys_mremap, "mremap", old_address, old_size, new_size, flags, new_address); +} diff --git a/src/musl/msync.cpp b/src/musl/msync.cpp index b69db69ceb..e2db34c97e 100644 --- a/src/musl/msync.cpp +++ b/src/musl/msync.cpp @@ -1,7 +1,12 @@ #include "common.hpp" +static long sys_msync(void */*addr*/, size_t /*length*/, int /*flags*/) +{ + return -ENOSYS; +} + extern "C" -long syscall_SYS_msync() { - STUB("msync"); - return 0; +long syscall_SYS_msync(void *addr, size_t length, int flags) +{ + return strace(sys_msync, "msync", addr, length, flags); } diff --git a/src/musl/munmap.cpp b/src/musl/munmap.cpp index 7af8ef3814..89c56eaa5d 100644 --- a/src/musl/munmap.cpp +++ b/src/musl/munmap.cpp @@ -1,4 +1,4 @@ -#include "stub.hpp" +#include "common.hpp" extern "C" void kfree(void* addr, size_t length); @@ -14,5 +14,5 @@ static long sys_munmap(void *addr, size_t length) extern "C" long syscall_SYS_munmap(void *addr, size_t length) { - return stubtrace(sys_munmap, "munmap", addr, length); + return strace(sys_munmap, "munmap", addr, length); } diff --git a/src/musl/nanosleep.cpp b/src/musl/nanosleep.cpp index d8adeae924..06b2a67e29 100644 --- a/src/musl/nanosleep.cpp +++ b/src/musl/nanosleep.cpp @@ -1,7 +1,13 @@ #include "common.hpp" +#include + +static long sys_nanosleep(const struct timespec */*req*/, struct timespec */*rem*/) +{ + return -ENOSYS; +} extern "C" -long syscall_SYS_nanosleep() { - STUB("nanosleep"); - return 0; +long syscall_SYS_nanosleep(const struct timespec *req, struct timespec *rem) +{ + return strace(sys_nanosleep, "nanosleep", req, rem); } diff --git a/src/musl/pipe.cpp b/src/musl/pipe.cpp index ad2635c209..2fbdae0e0e 100644 --- a/src/musl/pipe.cpp +++ b/src/musl/pipe.cpp @@ -1,21 +1,21 @@ -#include "stub.hpp" +#include "common.hpp" -static long sys_pipe() +static long sys_pipe([[maybe_unused]]int pipefd[2]) { - return 0; + return -ENOSYS; } -static long sys_pipe2() +static long sys_pipe2([[maybe_unused]]int pipefd[2], [[maybe_unused]]int flags) { - return 0; + return -ENOSYS; } extern "C" { -long syscall_SYS_pipe() { - return stubtrace(sys_pipe, "pipe"); +long syscall_SYS_pipe(int pipefd[2]) { + return strace(sys_pipe, "pipe", pipefd); } -long syscall_SYS_pipe2() { - return stubtrace(sys_pipe2, "pipe2"); +long syscall_SYS_pipe2(int pipefd[2], int flags) { + return strace(sys_pipe2, "pipe2", pipefd, flags); } } // extern "C" diff --git a/src/musl/prlimit64.cpp b/src/musl/prlimit64.cpp index 0528e372d7..7cfb68812a 100644 --- a/src/musl/prlimit64.cpp +++ b/src/musl/prlimit64.cpp @@ -1,11 +1,17 @@ -#include "stub.hpp" +#include "common.hpp" +#include -static long sys_prlimit64() +static long sys_prlimit64(pid_t /*pid*/, int /*resource*/, + const struct rlimit */*new_limit*/, + struct rlimit */*old_limit*/) { - return 0; + return -ENOSYS; } extern "C" -long syscall_SYS_prlimit64() { - return stubtrace(sys_prlimit64, "prlimit64"); +long syscall_SYS_prlimit64(pid_t pid, int resource, + const struct rlimit *new_limit, + struct rlimit *old_limit) +{ + return strace(sys_prlimit64, "prlimit64", pid, resource, new_limit, old_limit); } diff --git a/src/musl/rename.cpp b/src/musl/rename.cpp index df26423e48..b877116662 100644 --- a/src/musl/rename.cpp +++ b/src/musl/rename.cpp @@ -1,13 +1,14 @@ -#include "stub.hpp" +#include "common.hpp" #include static long sys_rename(const char* /*oldpath*/, const char* /*newpath*/) { - return -ENOSYS; + // currently makes no sense, especially since we're read-only + return -EROFS; } extern "C" long syscall_SYS_rename(const char* oldpath, const char* newpath) { - return stubtrace(sys_rename, "rename", oldpath, newpath); + return strace(sys_rename, "rename", oldpath, newpath); } diff --git a/src/musl/sched_getaffinity.cpp b/src/musl/sched_getaffinity.cpp index e2a8e4d46b..225feb5044 100644 --- a/src/musl/sched_getaffinity.cpp +++ b/src/musl/sched_getaffinity.cpp @@ -1,11 +1,15 @@ -#include "stub.hpp" +#include "common.hpp" +#include -static long sys_sched_getaffinity() +static long sys_sched_getaffinity(pid_t /*pid*/, size_t /*cpusetsize*/, + cpu_set_t */*mask*/) { - return 0; + return -ENOSYS; } extern "C" -long syscall_SYS_sched_getaffinity() { - return stubtrace(sys_sched_getaffinity, "sched_getaffinity"); +long syscall_SYS_sched_getaffinity(pid_t pid, size_t cpusetsize, + cpu_set_t *mask) +{ + return strace(sys_sched_getaffinity, "sched_getaffinity", pid, cpusetsize, mask); } diff --git a/src/musl/set_robust_list.cpp b/src/musl/set_robust_list.cpp index a1e180c590..69465e6537 100644 --- a/src/musl/set_robust_list.cpp +++ b/src/musl/set_robust_list.cpp @@ -1,11 +1,11 @@ -#include "stub.hpp" +#include "common.hpp" -static long sys_set_robust_list() +static long sys_set_robust_list(struct robust_list_head */*head*/, size_t /*len*/) { - return 0; + return -ENOSYS; } extern "C" -long syscall_SYS_set_robust_list() { - return stubtrace(sys_set_robust_list, "set_robust_list"); +long syscall_SYS_set_robust_list(struct robust_list_head *head, size_t len) { + return strace(sys_set_robust_list, "set_robust_list", head, len); } diff --git a/src/musl/sysinfo.cpp b/src/musl/sysinfo.cpp index 06c5024cd0..4c58bee708 100644 --- a/src/musl/sysinfo.cpp +++ b/src/musl/sysinfo.cpp @@ -1,11 +1,12 @@ -#include "stub.hpp" +#include "common.hpp" +#include -static long sys_sysinfo() +static long sys_sysinfo(struct sysinfo */*info*/) { - return 0; + return -ENOSYS; } extern "C" -long syscall_SYS_sysinfo() { - return stubtrace(sys_sysinfo, "sysinfo"); +long syscall_SYS_sysinfo(struct sysinfo *info) { + return strace(sys_sysinfo, "sysinfo", info); } From 2b09e84de1884a1e7e500db38635285021be87a2 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 28 Aug 2018 09:00:46 +0200 Subject: [PATCH 556/723] linux: Update TCP m2m benchmark --- test/linux/tcp/service.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/test/linux/tcp/service.cpp b/test/linux/tcp/service.cpp index 8fc4277548..9e336e2ec9 100644 --- a/test/linux/tcp/service.cpp +++ b/test/linux/tcp/service.cpp @@ -17,11 +17,13 @@ #include // rand() #include +#include #include #include #include #include #include "../router/async_device.hpp" +#define ENABLE_JUMBO_FRAMES static const size_t CHUNK_SIZE = 1024 * 1024; static const size_t NUM_CHUNKS = 2048; @@ -76,16 +78,24 @@ void Service::start() printf("Server reveived %zu Mb in %f sec. - %f Mbps \n", count_bytes / (1024 * 1024), time_sec, mbps); + + for (const auto& stat : Statman::get()) + { + printf("-> %s: %s\n", stat.name(), stat.to_string().c_str()); + } OS::shutdown(); } }); }); - printf("*** Linux userspace tcp demo started ***\n"); + printf("*** Linux userspace TCP demo started ***\n"); + printf("Measuring memory <-> memory bandwidth...\n"); time_start = now(); - inet_client.tcp().connect({{"10.0.0.42"}, 80}, [buf](auto conn){ + inet_client.tcp().connect({net::ip4::Addr{"10.0.0.42"}, 80}, + [buf](auto conn) + { if (not conn) std::abort(); @@ -93,3 +103,13 @@ void Service::start() conn->write(buf); }); } + +#ifdef ENABLE_JUMBO_FRAMES +#include +namespace hw { + uint16_t Nic::MTU_detection_override(int idx, const uint16_t default_MTU) + { + return 9000; + } +} +#endif From fea5a16c8225941cfd5a5d406b8a7abead1c04cb Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 28 Aug 2018 11:12:30 +0200 Subject: [PATCH 557/723] cxxabi: More complete ubsan output --- src/crt/cxx_abi.cpp | 67 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/src/crt/cxx_abi.cpp b/src/crt/cxx_abi.cpp index 10bfd53443..c4800230e4 100644 --- a/src/crt/cxx_abi.cpp +++ b/src/crt/cxx_abi.cpp @@ -64,20 +64,57 @@ extern "C" return 'x'; } + struct source_location { + const char *file_name; + struct { + uint32_t line; + uint32_t column; + }; + }; + struct out_of_bounds { + struct source_location src; + //struct type_descriptor *array_type; + //struct type_descriptor *index_type; + }; + struct overflow { + source_location src; + }; + struct mismatch { + source_location src; + }; + struct nonnull_return { + source_location src; + source_location attr; + }; + struct unreachable { + source_location src; + }; using ub_error = std::runtime_error; - void undefined_throw(const char*) { - + void print_src_location(const source_location& src) { + printf("ubsan: %s at line %u col %u\n", + src.file_name, src.line, src.column); + } + void undefined_throw(const char* error) { + printf("ubsan: %s", error); + print_backtrace(); + printf("\n"); } /// Undefined-behavior sanitizer - void __ubsan_handle_out_of_bounds() + void __ubsan_handle_out_of_bounds(struct out_of_bounds* data) { + print_src_location(data->src); undefined_throw("Out-of-bounds access"); } void __ubsan_handle_missing_return() { undefined_throw("Missing return"); } + void __ubsan_handle_nonnull_return(struct nonnull_return* data) + { + print_src_location(data->src); + undefined_throw("Non-null return"); + } void __ubsan_handle_add_overflow() { @@ -99,8 +136,12 @@ extern "C" { undefined_throw("Pointer overflow"); } - void __ubsan_handle_divrem_overflow() + void __ubsan_handle_divrem_overflow(struct overflow* data, + unsigned long lhs, + unsigned long rhs) { + print_src_location(data->src); + printf("ubsan: LHS %lu / RHS %lu\n", lhs, rhs); undefined_throw("Division remainder overflow"); } void __ubsan_handle_float_cast_overflow() @@ -112,20 +153,32 @@ extern "C" undefined_throw("Shift out-of-bounds"); } - void __ubsan_handle_type_mismatch_v1() + void __ubsan_handle_type_mismatch_v1(struct mismatch* data, unsigned long ptr) { - undefined_throw("Type mismatch"); + print_src_location(data->src); + char buffer[1024]; + snprintf(buffer, sizeof(buffer), + "Type mismatch on ptr %p, %s", + (void*) ptr, + (ptr < 1024) ? "was nullptr deref" : "normal"); + undefined_throw(buffer); } void __ubsan_handle_function_type_mismatch() { undefined_throw("Function type mismatch"); } + void __ubsan_handle_invalid_builtin() + { + undefined_throw("Invalid built-in function"); + } void __ubsan_handle_load_invalid_value() { undefined_throw("Load of invalid value"); } - void __ubsan_handle_builtin_unreachable() + [[noreturn]] + void __ubsan_handle_builtin_unreachable(struct unreachable* data) { + print_src_location(data->src); panic("Unreachable code reached"); } } From 306e73266bcd1226c4c03d51f1bbc3c6c2907766 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 28 Aug 2018 11:56:14 +0200 Subject: [PATCH 558/723] mod: Update rapidjson to latest --- mod/rapidjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mod/rapidjson b/mod/rapidjson index b596f4e990..73063f5002 160000 --- a/mod/rapidjson +++ b/mod/rapidjson @@ -1 +1 @@ -Subproject commit b596f4e99013dcc2d15f3c71124f15e4330beb3a +Subproject commit 73063f5002612c6bf64fe24f851cd5cc0d83eef9 From 09f2e5fd2f82fc1d894cf6d7e59d4e0c6286d841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 28 Aug 2018 11:57:17 +0200 Subject: [PATCH 559/723] mana: Remove rapidjson warning by not throwing exception --- examples/acorn/app/models/squirrel.hpp | 13 ++++++++++++- examples/acorn/app/routes/squirrels.hpp | 12 +++++++----- lib/mana/include/mana/attributes/json.hpp | 14 -------------- lib/mana/src/middleware/parsley.cpp | 18 +++++++----------- 4 files changed, 26 insertions(+), 31 deletions(-) diff --git a/examples/acorn/app/models/squirrel.hpp b/examples/acorn/app/models/squirrel.hpp index ed4e0ad1b8..451d8eb260 100644 --- a/examples/acorn/app/models/squirrel.hpp +++ b/examples/acorn/app/models/squirrel.hpp @@ -152,10 +152,21 @@ inline void Squirrel::serialize(rapidjson::Writer& writ writer.EndObject(); } -inline bool Squirrel::deserialize(const rapidjson::Document& doc) { +inline bool Squirrel::deserialize(const rapidjson::Document& doc) +{ + if(not (doc.HasMember("name") and doc["name"].IsString())) + return false; + + if(not (doc.HasMember("age") and doc["age"].IsInt())) + return false; + + if(not (doc.HasMember("occupation") and doc["occupation"].IsString())) + return false; + name_ = doc["name"].GetString(); age_ = doc["age"].GetUint(); occupation_ = doc["occupation"].GetString(); + return true; } diff --git a/examples/acorn/app/routes/squirrels.hpp b/examples/acorn/app/routes/squirrels.hpp index 83418ff0f0..5812027c24 100644 --- a/examples/acorn/app/routes/squirrels.hpp +++ b/examples/acorn/app/routes/squirrels.hpp @@ -60,7 +60,13 @@ class Squirrels : public mana::Router { // create an empty model acorn::Squirrel s; // deserialize it - s.deserialize(doc); + bool ok = s.deserialize(doc); + if(UNLIKELY(not ok)) + { + printf("[Squirrels@POST:/] Could not deserialize squirrel\n"); + res->error({"Parsing Error", "Could not parse data."}); + return; + } // add to bucket auto id = squirrels->capture(s); assert(id == s.key); @@ -74,10 +80,6 @@ class Squirrels : public mana::Router { // send the created entity as response res->send_json(s.json()); } - catch(const Assert_error& e) { - printf("[Squirrels@POST:/] Assert_error: %s\n", e.what()); - res->error({"Parsing Error", "Could not parse data."}); - } catch(const bucket::ConstraintException& e) { printf("[Squirrels@POST:/] ConstraintException: %s\n", e.what()); res->error({"Constraint Exception", e.what()}); diff --git a/lib/mana/include/mana/attributes/json.hpp b/lib/mana/include/mana/attributes/json.hpp index bb834aeb09..fd28ccaaa6 100644 --- a/lib/mana/include/mana/attributes/json.hpp +++ b/lib/mana/include/mana/attributes/json.hpp @@ -26,20 +26,6 @@ #define RAPIDJSON_THROWPARSEEXCEPTION 1 #endif -#include -/** - * - */ -namespace mana { - - struct Assert_error : public std::logic_error { - using std::logic_error::logic_error; - }; //< struct Assert_error - -} // < namespace mana - -#define RAPIDJSON_ASSERT(x) if (!(x)) throw mana::Assert_error(RAPIDJSON_STRINGIFY(x)) - #include #include #include diff --git a/lib/mana/src/middleware/parsley.cpp b/lib/mana/src/middleware/parsley.cpp index 6e3cc13a6d..c8cddf6741 100644 --- a/lib/mana/src/middleware/parsley.cpp +++ b/lib/mana/src/middleware/parsley.cpp @@ -31,19 +31,15 @@ void Parsley::process(mana::Request_ptr req, mana::Response_ptr, mana::Next next auto json = std::make_shared(); // Access the document and parse the body - try { - json->doc().Parse(req->source().body().data()); - #ifdef VERBOSE_WEBSERVER - printf(" Parsed JSON data.\n"); - #endif + bool err = json->doc().Parse(req->source().body().data()).HasParseError(); + #ifdef VERBOSE_WEBSERVER + printf(" Parsed JSON data.\n"); + #endif - // Add the json attribute to the request + if(not err) req->set_attribute(std::move(json)); - } - catch(const Assert_error& e) { - printf(" Parsing error.\n"); - } - + else + printf(" Parsing error\n"); } return (*next)(); From e874c9851df25a7cae16c9d8e99211e41b6f971d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 28 Aug 2018 15:31:36 +0200 Subject: [PATCH 560/723] ubsan: Add better type mismatch support --- src/crt/cxx_abi.cpp | 31 ++++++++++++++++++++++++------ src/platform/x86_pc/CMakeLists.txt | 5 +++-- src/platform/x86_pc/smbios.cpp | 4 ++-- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/crt/cxx_abi.cpp b/src/crt/cxx_abi.cpp index c4800230e4..a877050731 100644 --- a/src/crt/cxx_abi.cpp +++ b/src/crt/cxx_abi.cpp @@ -71,16 +71,25 @@ extern "C" uint32_t column; }; }; + struct type_descriptor { + uint16_t type_kind; + uint16_t type_info; + char type_name[1]; + }; + struct out_of_bounds { - struct source_location src; - //struct type_descriptor *array_type; - //struct type_descriptor *index_type; + source_location src; + type_descriptor* array_type; + type_descriptor* index_type; }; struct overflow { source_location src; }; struct mismatch { - source_location src; + source_location src; + type_descriptor* type; + unsigned char log_align; + unsigned char type_kind; }; struct nonnull_return { source_location src; @@ -156,11 +165,21 @@ extern "C" void __ubsan_handle_type_mismatch_v1(struct mismatch* data, unsigned long ptr) { print_src_location(data->src); + const char* reason = "Type mismatch"; + const long alignment = 1 << data->log_align; + + if (alignment && (ptr & (alignment-1)) != 0) { + reason = "Misaligned access"; + } + else if (ptr == 0) { + reason = "Null-pointer access"; + } char buffer[1024]; snprintf(buffer, sizeof(buffer), - "Type mismatch on ptr %p, %s", + "%s on ptr %p (aligned %lu)", + reason, (void*) ptr, - (ptr < 1024) ? "was nullptr deref" : "normal"); + alignment); undefined_throw(buffer); } void __ubsan_handle_function_type_mismatch() diff --git a/src/platform/x86_pc/CMakeLists.txt b/src/platform/x86_pc/CMakeLists.txt index 0ec8a61b45..9ddc945567 100644 --- a/src/platform/x86_pc/CMakeLists.txt +++ b/src/platform/x86_pc/CMakeLists.txt @@ -38,8 +38,9 @@ add_library(x86_pc STATIC ${X86_PC_OBJECTS} apic_boot.o) add_dependencies(x86_pc PrecompiledLibraries) # disable sanitizers on kernel_start and others -set_source_files_properties(kernel_start.cpp PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") -set_source_files_properties(serial1.cpp PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") +#set_source_files_properties(kernel_start.cpp PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") +#set_source_files_properties(serial1.cpp PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") +set_source_files_properties(smbios.cpp PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") add_subdirectory(boot) set_target_properties(x86_pc PROPERTIES LINKER_LANGUAGE CXX) diff --git a/src/platform/x86_pc/smbios.cpp b/src/platform/x86_pc/smbios.cpp index 61a471347f..d4616be9bf 100644 --- a/src/platform/x86_pc/smbios.cpp +++ b/src/platform/x86_pc/smbios.cpp @@ -50,7 +50,7 @@ namespace x86 uint8_t length; uint16_t handle; - char data[0]; + const char data[0]; const char* strings() const { @@ -74,7 +74,7 @@ namespace x86 { int len = strnlen(str, 64); str += len + 1; - if (len == 0) return (Header*) str; + if (len == 0) return (const Header*) str; } } }; From 75cdbb62288df12ee78486c3bec1606c391eedb4 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 28 Aug 2018 15:32:42 +0200 Subject: [PATCH 561/723] vmxnet3: Fix misaligment spreading to Packets --- src/drivers/vmxnet3.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index 5a725a85df..16cb65c917 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -110,8 +110,9 @@ inline void mmio_write32(uintptr_t location, uint32_t value) static inline uint16_t buffer_size_for_mtu(const uint16_t mtu) { const uint16_t header = sizeof(net::Packet) + vmxnet3::DRIVER_OFFSET; - const uint16_t total = header + sizeof(net::ethernet::VLAN_header) + mtu; - if (total <= 2048) return 2048; + uint16_t total = header + sizeof(net::ethernet::VLAN_header) + mtu; + if (total & 15) total += 16 - (total & 15); + //if (total & 2047) total += 2048 - (total & 2047); assert(total <= 16384 && "Buffers larger than 16k are not supported"); return total; } From 83681226a5152e1c9a0868f7d960a0b1d29f3ef1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 28 Aug 2018 15:45:20 +0200 Subject: [PATCH 562/723] ubsan: Add back SMBIOS, fix misalignment --- src/platform/x86_pc/CMakeLists.txt | 2 +- src/platform/x86_pc/smbios.cpp | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/platform/x86_pc/CMakeLists.txt b/src/platform/x86_pc/CMakeLists.txt index 9ddc945567..98dc55126b 100644 --- a/src/platform/x86_pc/CMakeLists.txt +++ b/src/platform/x86_pc/CMakeLists.txt @@ -40,7 +40,7 @@ add_dependencies(x86_pc PrecompiledLibraries) # disable sanitizers on kernel_start and others #set_source_files_properties(kernel_start.cpp PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") #set_source_files_properties(serial1.cpp PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") -set_source_files_properties(smbios.cpp PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") +#set_source_files_properties(smbios.cpp PROPERTIES COMPILE_FLAGS "-fno-sanitize=all") add_subdirectory(boot) set_target_properties(x86_pc PROPERTIES LINKER_LANGUAGE CXX) diff --git a/src/platform/x86_pc/smbios.cpp b/src/platform/x86_pc/smbios.cpp index d4616be9bf..d53ecaa7e8 100644 --- a/src/platform/x86_pc/smbios.cpp +++ b/src/platform/x86_pc/smbios.cpp @@ -77,7 +77,7 @@ namespace x86 if (len == 0) return (const Header*) str; } } - }; + } __attribute__((packed)); struct PhysMemArray : public Header { @@ -102,8 +102,7 @@ namespace x86 return (inf.capacity32 == 0x80000000) ? inf.capacity64 : inf.capacity32 * 1024; } - - }; + } __attribute__((packed)); void SMBIOS::parse(const char* mem) { From ee784b37bdacffea4c09831bda7b39c804afef82 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Wed, 29 Aug 2018 07:35:54 +0200 Subject: [PATCH 563/723] Git describe: Added --tags to all git describe commands --- CMakeLists.txt | 2 +- linux/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fa66f78874..757d9d350b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -32,7 +32,7 @@ set(SCRIPTS ${CMAKE_INSTALL_PREFIX}/includeos/scripts) set(CPP_VERSION c++17) # create OS version string from git describe (used in CXX flags) -execute_process(COMMAND git describe --dirty +execute_process(COMMAND git describe --tags --dirty WORKING_DIRECTORY ${INCLUDEOS_ROOT} OUTPUT_VARIABLE OS_VERSION) string(STRIP ${OS_VERSION} OS_VERSION) diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index 7226c88ebb..c8290d492b 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -2,7 +2,7 @@ cmake_minimum_required (VERSION 3.0.2) project (userland_tests C CXX) # create OS version string from git describe (used in CXX flags) -execute_process(COMMAND git describe --dirty +execute_process(COMMAND git describe --tags --dirty WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/.. OUTPUT_VARIABLE OS_VERSION) string(STRIP ${OS_VERSION} OS_VERSION) From 7c2f983865ca3ea72d5c67ee19c276f0fa403049 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 29 Aug 2018 10:09:16 +0200 Subject: [PATCH 564/723] ubsan: Resolve addresses and elaborate function type mismatch --- src/crt/cxx_abi.cpp | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/src/crt/cxx_abi.cpp b/src/crt/cxx_abi.cpp index a877050731..e96f946041 100644 --- a/src/crt/cxx_abi.cpp +++ b/src/crt/cxx_abi.cpp @@ -19,6 +19,7 @@ #include #include #include +#include /** * This header is for instantiating and implementing @@ -76,6 +77,10 @@ extern "C" uint16_t type_info; char type_name[1]; }; + const char* type_kind_string[] = { + "load of", "store to", "reference binding to", "member access within", + "member call on", "constructor call on", "downcast of", "downcast of" + }; struct out_of_bounds { source_location src; @@ -87,10 +92,14 @@ extern "C" }; struct mismatch { source_location src; - type_descriptor* type; + type_descriptor& type; unsigned char log_align; unsigned char type_kind; }; + struct function_type_mismatch { + const source_location src; + const type_descriptor& type; + }; struct nonnull_return { source_location src; source_location attr; @@ -165,6 +174,9 @@ extern "C" void __ubsan_handle_type_mismatch_v1(struct mismatch* data, unsigned long ptr) { print_src_location(data->src); + char sbuf[1024]; + auto res = Elf::safe_resolve_symbol((void*) ptr, sbuf, sizeof(sbuf)); + const char* reason = "Type mismatch"; const long alignment = 1 << data->log_align; @@ -174,17 +186,35 @@ extern "C" else if (ptr == 0) { reason = "Null-pointer access"; } - char buffer[1024]; + char buffer[2048]; snprintf(buffer, sizeof(buffer), - "%s on ptr %p (aligned %lu)", + "%s on ptr %p (aligned %lu)\n" + "ubsan: type name %s\n" + "ubsan: symbol %s", reason, (void*) ptr, - alignment); + alignment, + data->type.type_name, + res.name); undefined_throw(buffer); } - void __ubsan_handle_function_type_mismatch() + void __ubsan_handle_function_type_mismatch( + struct function_type_mismatch* data, + unsigned long ptr) { - undefined_throw("Function type mismatch"); + print_src_location(data->src); + char sbuf[1024]; + auto res = Elf::safe_resolve_symbol((void*) ptr, sbuf, sizeof(sbuf)); + + char buffer[2048]; + snprintf(buffer, sizeof(buffer), + "Function type mismatch on ptr %p\n" + "ubsan: type name %s\n" + "ubsan: function %s", + (void*) ptr, + data->type.type_name, + res.name); + undefined_throw(buffer); } void __ubsan_handle_invalid_builtin() { From 1bfcf8bde5a2aff6420c1f4d4a53b0c8771d2a2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Wed, 29 Aug 2018 15:56:30 +0200 Subject: [PATCH 565/723] uplink: Store index from config, move serialization to Config --- lib/uplink/config.cpp | 49 +++++++++++++++++++++++++++++++++++++--- lib/uplink/config.hpp | 7 +++++- lib/uplink/ws_uplink.cpp | 21 ++--------------- 3 files changed, 54 insertions(+), 23 deletions(-) diff --git a/lib/uplink/config.cpp b/lib/uplink/config.cpp index 62cb3fc495..fcaaaab98a 100644 --- a/lib/uplink/config.cpp +++ b/lib/uplink/config.cpp @@ -69,17 +69,17 @@ namespace uplink { if(index.IsNumber()) { - config_.inet = &net::Super_stack::get(index.GetInt()); + config_.index = index.GetInt(); } else { - config_.inet = &net::Super_stack::get(index.GetString()); + config_.index_str = index.GetString(); } } // If not given, pick the first stack else { - config_.inet = &net::Super_stack::get(0); + config_.index = 0; } // Reboot on panic (optional) @@ -118,4 +118,47 @@ namespace uplink { return config_; } + std::string Config::serialized_string() const + { + using namespace rapidjson; + StringBuffer buf; + Writer writer{buf}; + + writer.StartObject(); + + writer.Key("index"); + (index >= 0) ? writer.Int(index) : writer.String(index_str); + + writer.Key("url"); + writer.String(url); + + writer.Key("token"); + writer.String(token); + + writer.Key("certs"); + writer.String(certs_path); + + writer.Key("verify"); + writer.Bool(verify_certs); + + writer.Key("reboot"); + writer.Bool(reboot); + + writer.Key("ws_logging"); + writer.Bool(ws_logging); + + writer.Key("serialize_ct"); + writer.Bool(serialize_ct); + + writer.EndObject(); + + return buf.GetString(); + } + + net::Inet& Config::get_stack() const + { + return (index >= 0) ? net::Super_stack::get(index) + : net::Super_stack::get(index_str); + } + } diff --git a/lib/uplink/config.hpp b/lib/uplink/config.hpp index 3be19041dd..c00743532c 100644 --- a/lib/uplink/config.hpp +++ b/lib/uplink/config.hpp @@ -29,7 +29,8 @@ namespace uplink { struct Config { - net::Inet *inet; + std::string index_str; + int index{-1}; uri::URI url; std::string token; std::string tag; @@ -40,6 +41,10 @@ namespace uplink { bool serialize_ct = false; static Config read(); + + std::string serialized_string() const; + + net::Inet& get_stack() const; }; } diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index d0c0439752..12e5e49589 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -75,7 +75,7 @@ namespace uplink { WS_uplink::WS_uplink(Config config) : config_{std::move(config)}, - inet_{*config_.inet}, + inet_{config_.get_stack()}, id_{__arch_system_info().uuid}, parser_({this, &WS_uplink::handle_transport}), heartbeat_timer({this, &WS_uplink::on_heartbeat_timer}) @@ -536,25 +536,8 @@ namespace uplink { void WS_uplink::send_uplink() { MYINFO("[ %s ] Sending uplink", isotime::now().c_str()); - using namespace rapidjson; - - StringBuffer buf; - Writer writer{buf}; - - writer.StartObject(); - writer.Key("url"); - writer.String(config_.url); - - writer.Key("token"); - writer.String(config_.token); - - writer.Key("reboot"); - writer.Bool(config_.reboot); - - writer.EndObject(); - - std::string str = buf.GetString(); + auto str = config_.serialized_string(); MYINFO("%s", str.c_str()); From 714cd0672ad56aeefd4fdb7fe027021b9d8ba59a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 29 Aug 2018 16:25:33 +0200 Subject: [PATCH 566/723] ubsan: GCC support, add more inspection for ELF symbols --- src/crt/c_abi.c | 1 + src/crt/cxx_abi.cpp | 10 +++++-- src/kernel/elf.cpp | 41 ++++++++++++++++++++++++++++ src/platform/x86_pc/kernel_start.cpp | 9 +++--- src/platform/x86_pc/serial1.cpp | 2 +- 5 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/crt/c_abi.c b/src/crt/c_abi.c index 9a7df03c1e..7a7cf63ed3 100644 --- a/src/crt/c_abi.c +++ b/src/crt/c_abi.c @@ -24,6 +24,7 @@ extern __typeof (name) aliasname __attribute__ ((weak, alias (#name))); void* __dso_handle; +__attribute__((no_sanitize("all"))) uint32_t _move_symbols(void* sym_loc) { // re-align new symbol area to a page boundary diff --git a/src/crt/cxx_abi.cpp b/src/crt/cxx_abi.cpp index e96f946041..c63746fcb3 100644 --- a/src/crt/cxx_abi.cpp +++ b/src/crt/cxx_abi.cpp @@ -20,6 +20,7 @@ #include #include #include +#include /** * This header is for instantiating and implementing @@ -113,9 +114,9 @@ extern "C" src.file_name, src.line, src.column); } void undefined_throw(const char* error) { - printf("ubsan: %s", error); + kprintf("ubsan: %s", error); print_backtrace(); - printf("\n"); + kprintf("\n"); } /// Undefined-behavior sanitizer @@ -216,6 +217,11 @@ extern "C" res.name); undefined_throw(buffer); } + void __ubsan_handle_nonnull_arg() + { + undefined_throw("Non-null argument violated"); + } + void __ubsan_handle_invalid_builtin() { undefined_throw("Invalid built-in function"); diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index 15d799c707..8e132459af 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -435,6 +435,47 @@ void _init_elf_parser() } } +extern "C" +void __elf_validate_section(const void* location) +{ + int size = _get_elf_section_datasize(location); + // stripped variant + if (size == 0) { + kprintf("ELF syms are considered stripped\n"); + asm("cli; hlt"); + } + // incoming header + auto* hdr = (elfsyms_header*) location; + // verify CRC sanity check + const uint32_t temp_hdr = hdr->sanity_check; + hdr->sanity_check = 0; + const uint32_t our_sanity = crc32c(hdr, sizeof(elfsyms_header)); + hdr->sanity_check = temp_hdr; + if (hdr->sanity_check != our_sanity) + { + kprintf("ELF syms header CRC failed! " + "(%08x vs %08x)\n", hdr->sanity_check, our_sanity); + asm("cli; hlt"); + } + + // verify separate checksums of symbols and strings + uint32_t symbsize = hdr->symtab_entries * sizeof(ElfSym); + uint32_t csum_syms = crc32c(hdr->syms, symbsize); + uint32_t csum_strs = crc32c(&hdr->syms[hdr->symtab_entries], hdr->strtab_size); + if (csum_syms != hdr->checksum_syms || csum_strs != hdr->checksum_strs) + { + if (csum_syms != hdr->checksum_syms) + kprintf("ELF symbol tables checksum failed! " + "(%08x vs %08x)\n", csum_syms, hdr->checksum_syms); + if (csum_strs != hdr->checksum_strs) + kprintf("ELF string tables checksum failed! " + "(%08x vs %08x)\n", csum_strs, hdr->checksum_strs); + uint32_t all = crc32c(hdr, sizeof(elfsyms_header) + size); + kprintf("Checksum ELF section: %08x\n", all); + asm("cli; hlt"); + } +} + #ifdef ARCH_x86_64 #include void elf_protect_symbol_areas() diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 2e39935b45..e0c0d4fb50 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -62,6 +62,7 @@ uint32_t __grub_addr = 0x7001; static volatile int __global_ctors_ok = 0; +__attribute__((no_sanitize("all"))) void _init_bss() { extern char _BSS_START_, _BSS_END_; @@ -75,6 +76,7 @@ static void global_ctor_test(){ int kernel_main(int, char * *, char * *) { PRATTLE(" libc initialization complete \n"); + kernel_sanity_checks(); Expects(__global_ctors_ok == 42); extern bool __libc_initialized; __libc_initialized = true; @@ -99,16 +101,12 @@ int kernel_main(int, char * *, char * *) { return 0; } -void __init_tls(size_t* p) -{ - kprintf("init_tls(%p)\n", p); -} - // Musl entry extern "C" int __libc_start_main(int (*main)(int,char **,char **), int argc, char **argv); extern "C" uintptr_t __syscall_entry(); +extern "C" void __elf_validate_section(const void*); extern "C" __attribute__((no_sanitize("all"))) @@ -243,6 +241,7 @@ void kernel_start(uint32_t magic, uint32_t addr) #warning Classical syscall interface missing for 32-bit #endif + kernel_sanity_checks(); // GDB_ENTRY; PRATTLE("* Starting libc initialization\n"); __libc_start_main(kernel_main, argc, argv.data()); diff --git a/src/platform/x86_pc/serial1.cpp b/src/platform/x86_pc/serial1.cpp index 4fbabb6712..789ee6c677 100644 --- a/src/platform/x86_pc/serial1.cpp +++ b/src/platform/x86_pc/serial1.cpp @@ -4,12 +4,12 @@ static const uint16_t port = 0x3F8; // Serial 1 static char initialized = 0xFF; extern "C" -__attribute__((no_sanitize("all"))) void __init_serial1() { initialized = false; } +__attribute__((no_sanitize("all"))) static inline void init_if_needed() { if (initialized) return; From 542f671725099cad5d2232ca626f2da50ee750f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 30 Aug 2018 09:19:16 +0200 Subject: [PATCH 567/723] uplink: Also serialize tag --- lib/uplink/config.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/uplink/config.cpp b/lib/uplink/config.cpp index fcaaaab98a..29bb569103 100644 --- a/lib/uplink/config.cpp +++ b/lib/uplink/config.cpp @@ -135,6 +135,9 @@ namespace uplink { writer.Key("token"); writer.String(token); + writer.Key("tag"); + writer.String(tag); + writer.Key("certs"); writer.String(certs_path); From c7b35dc65155dba4e1cd925483008eb9a9cd60bd Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 30 Aug 2018 12:46:32 +0200 Subject: [PATCH 568/723] cmake: Add various LTO options --- CMakeLists.txt | 7 +++++++ cmake/post.service.cmake | 11 +++++++++-- cmake/pre.service.cmake | 4 +++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fa66f78874..693e6c8c60 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,6 +53,8 @@ option(threading "Compile with SMP/threading support" OFF) option(silent "Disable most output during OS boot" OFF) option (undefined_san "Enable undefined-behavior sanitizer" OFF) +option (thin_lto "Enable the Thin LTO plugin" OFF) +option (full_lto "Enable full LTO (compatibility)" OFF) # create random hex string as stack protector canary string(RANDOM LENGTH 16 ALPHABET 0123456789ABCDEF STACK_PROTECTOR_VALUE) @@ -115,6 +117,11 @@ set(CAPABS "${CAPABS} ${OPTIMIZE}") if (undefined_san) set(CAPABS "${CAPABS} -fsanitize=undefined -fno-sanitize=vptr") endif() +if (thin_lto) + set(CAPABS "${CAPABS} -flto=thin") +elseif(full_lto) + set(CAPABS "${CAPABS} -flto=full") +endif() # object format needs to be set BEFORE enabling ASM # see: https://cmake.org/Bug/bug_relationship_graph.php?bug_id=13166 diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 13c6bc9ff6..3dc17e45ad 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -59,10 +59,17 @@ endif() if (undefined_san) set(CAPABS "${CAPABS} -fsanitize=undefined -fno-sanitize=vptr") endif() +if (thin_lto) + set(CMAKE_LINKER "ld.lld") + set(OPTIMIZE "${OPTIMIZE} -flto=thin -fuse-ld=lld") +elseif (full_lto) + set(CMAKE_LINKER "ld.lld") + set(OPTIMIZE "${OPTIMIZE} -flto=full") +endif() if (CMAKE_COMPILER_IS_GNUCC) - set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c -std=${CPP_VERSION}") - set(CMAKE_C_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c") + set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} ${OPTIMIZE} ${WARNS} -nostdlib -fno-omit-frame-pointer -c -std=${CPP_VERSION}") + set(CMAKE_C_FLAGS "-MMD ${CAPABS} ${OPTIMIZE} ${WARNS} -nostdlib -fno-omit-frame-pointer -c") else() # these kinda work with llvm set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} ${OPTIMIZE} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c -std=${CPP_VERSION} ") diff --git a/cmake/pre.service.cmake b/cmake/pre.service.cmake index be8d21e013..c92cce6fcf 100644 --- a/cmake/pre.service.cmake +++ b/cmake/pre.service.cmake @@ -23,7 +23,9 @@ option(minimal "Build for minimal size" OFF) option(stripped "Strip symbols to further reduce size" OFF) option(threading "Compile threading and SMP support" OFF) -option (undefined_san "Enable undefined-behavior sanitizer" OFF) +option(undefined_san "Enable undefined-behavior sanitizer" OFF) +option(thin_lto "Enable Thin LTO plugin" OFF) +option(full_lto "Enable full LTO (also works on LD)" OFF) option(coroutines "Compile with coroutines TS support" OFF) # arch and platform defines From 8d159b89cbadf747a91dc45545f112b469d62f3c Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 30 Aug 2018 14:26:47 +0200 Subject: [PATCH 569/723] vmxnet3: Support for 2 RX queues --- src/drivers/vmxnet3.cpp | 32 +++++++++++++++++--------------- src/drivers/vmxnet3.hpp | 2 +- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index 16cb65c917..df65790ba6 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -35,7 +35,7 @@ static std::vector deferred_devs; #define VMXNET3_NUM_TX_COMP vmxnet3::NUM_TX_DESC #define VMXNET3_NUM_RX_COMP vmxnet3::NUM_RX_DESC -static const int VMXNET3_TX_FILL = vmxnet3::NUM_TX_DESC-1; +static const int VMXNET3_TX_FILL = vmxnet3::NUM_TX_DESC; static const int VMXNET3_RX_FILL = vmxnet3::NUM_RX_DESC; /** @@ -49,9 +49,11 @@ struct vmxnet3_dma { struct vmxnet3_tx_desc tx_desc[vmxnet3::NUM_TX_DESC]; struct vmxnet3_tx_comp tx_comp[VMXNET3_NUM_TX_COMP]; /** RX ring */ - struct vmxnet3_rx_desc rx0_desc[vmxnet3::NUM_RX_DESC]; - struct vmxnet3_rx_desc rx1_desc[vmxnet3::NUM_RX_DESC]; - struct vmxnet3_rx_comp rx_comp[VMXNET3_NUM_RX_COMP]; + struct vmxnet3_rx { + struct vmxnet3_rx_desc desc[vmxnet3::NUM_RX_DESC]; + struct vmxnet3_rx_comp comp[VMXNET3_NUM_RX_COMP]; + }; + struct vmxnet3_rx rx[vmxnet3::NUM_RX_QUEUES]; /** Queue descriptors */ struct vmxnet3_queues queues; /** Shared area */ @@ -201,9 +203,9 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : for (int q = 0; q < NUM_RX_QUEUES; q++) { memset(rx[q].buffers, 0, sizeof(rx[q].buffers)); - rx[q].desc0 = &dma->rx0_desc[0]; - rx[q].desc1 = &dma->rx1_desc[0]; - rx[q].comp = &dma->rx_comp[0]; + rx[q].desc0 = &dma->rx[q].desc[0]; + rx[q].desc1 = nullptr; + rx[q].comp = &dma->rx[q].comp[0]; rx[q].index = q; auto& queue = queues.rx[q]; @@ -211,7 +213,7 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : queue.cfg.desc_address[1] = (uintptr_t) rx[q].desc1; queue.cfg.comp_address = (uintptr_t) rx[q].comp; queue.cfg.num_desc[0] = vmxnet3::NUM_RX_DESC; - queue.cfg.num_desc[1] = vmxnet3::NUM_RX_DESC; + queue.cfg.num_desc[1] = 0; queue.cfg.num_comp = VMXNET3_NUM_RX_COMP; queue.cfg.driver_data_len = sizeof(vmxnet3_rx_desc) + 2 * sizeof(vmxnet3_rx_desc); @@ -436,9 +438,7 @@ void vmxnet3::msix_recv_handler() { for (int q = 0; q < NUM_RX_QUEUES; q++) { - this->disable_intr(2 + q); this->receive_handler(q); - this->enable_intr(2 + q); } } @@ -481,12 +481,13 @@ bool vmxnet3::transmit_handler() bool vmxnet3::receive_handler(const int Q) { std::vector recvq; + this->disable_intr(2 + Q); while (true) { uint32_t idx = rx[Q].consumers % VMXNET3_NUM_RX_COMP; uint32_t gen = (rx[Q].consumers & VMXNET3_NUM_RX_COMP) ? 0 : VMXNET3_RXCF_GEN; - auto& comp = dma->rx_comp[idx]; + auto& comp = dma->rx[Q].comp[idx]; // break when exiting this generation if (gen != (comp.flags & VMXNET3_RXCF_GEN)) break; rx[Q].consumers++; @@ -500,13 +501,14 @@ bool vmxnet3::receive_handler(const int Q) recvq.push_back(recv_packet(rx[Q].buffers[desc], len)); rx[Q].buffers[desc] = nullptr; } + this->enable_intr(2 + Q); // refill always if (!recvq.empty()) { this->refill(rx[Q]); - // handle_magic() - for (auto& pckt : recvq) { - Link::receive(std::move(pckt)); - } + } + // handle packets + for (auto& pckt : recvq) { + Link::receive(std::move(pckt)); } return recvq.empty() == false; } diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index a1ea2d552c..7479674099 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -31,7 +31,7 @@ class vmxnet3 : public net::Link_layer static const int DRIVER_OFFSET = 2; static const int NUM_RX_QUEUES = 1; static const int NUM_TX_DESC = 512; - static const int NUM_RX_DESC = 256; + static const int NUM_RX_DESC = 512; static std::unique_ptr new_instance(hw::PCI_Device& d, const uint16_t MTU) { return std::make_unique(d, MTU); } From 3136352aa0d7dd3263050a8b5d1f1aecf3d5a836 Mon Sep 17 00:00:00 2001 From: Ingve Vormestrand Date: Thu, 30 Aug 2018 14:50:30 +0200 Subject: [PATCH 570/723] examples: remove unneeded includes from http_client service example --- examples/http_client/service.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index f710de2250..f49cd7608a 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -36,8 +36,6 @@ static SSL_CTX* init_ssl_context() #include #include -#include -#include static void begin_http(net::Inet& inet) { From f4d501dc7ff514a2903659c8d34047525966ebbd Mon Sep 17 00:00:00 2001 From: Ingve Vormestrand Date: Thu, 30 Aug 2018 15:04:50 +0200 Subject: [PATCH 571/723] examples: add gratuitous C++14 to http_client service example --- examples/http_client/service.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index f49cd7608a..1edd8717af 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -36,6 +36,7 @@ static SSL_CTX* init_ssl_context() #include #include +using namespace std::literals::string_literals; static void begin_http(net::Inet& inet) { @@ -43,7 +44,7 @@ static void begin_http(net::Inet& inet) static Basic_client basic{inet.tcp()}; - const std::string url{"http://www.google.com"}; + const auto url{"http://www.google.com"s}; INFO("HTTP", "GET %s", url.c_str()); basic.get(url, {}, [url](Error err, Response_ptr res, Connection&) @@ -62,7 +63,7 @@ static void begin_http(net::Inet& inet) static Client client{inet.tcp(), ctx}; - const std::string url_sec{"https://www.google.com"}; + const auto url_sec{"https://www.google.com"s}; INFO("HTTPS", "(Secure) GET %s", url_sec.c_str()); client.get("https://www.google.com", {}, [url = url_sec](Error err, Response_ptr res, Connection&) From 3f1227a8366b9aa5edc2dcf4a6c762b0289f6ebf Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 30 Aug 2018 15:25:19 +0200 Subject: [PATCH 572/723] kernel: Fix timestamps output (somewhat) --- src/kernel/os.cpp | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index d23932d360..35abc8f5c7 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -191,23 +191,26 @@ void OS::print(const char* str, const size_t len) return; } + /** TIMESTAMPING **/ + if (OS::m_timestamps && OS::is_booted() && !OS::is_panicking()) + { + static bool apply_ts = true; + if (apply_ts) + { + std::string ts = "[" + isotime::now() + "] "; + for (auto& callback : os_print_handlers) { + callback(ts.c_str(), ts.size()); + } + apply_ts = false; + } + const bool has_newline = contains(str, len, '\n'); + if (has_newline) apply_ts = true; + } + /** TIMESTAMPING **/ + for (auto& callback : os_print_handlers) { if (os_enable_boot_logging || OS::is_booted() || OS::is_panicking()) { - if (OS::m_timestamps && OS::is_booted() && !OS::is_panicking()) - { - /** TIMESTAMPING **/ - static bool ts_shown = false; - if (ts_shown == false) - { - std::string ts = "[" + isotime::now() + "] "; - callback(ts.c_str(), ts.size()); - ts_shown = true; - } - const bool has_newline = contains(str, len, '\n'); - if (ts_shown && has_newline) ts_shown = false; - /** TIMESTAMPING **/ - } callback(str, len); } } From a70d08963f345a713b0247d78dd0f236c36d635d Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 31 Aug 2018 11:26:35 +0200 Subject: [PATCH 573/723] kernel: Enable stdout timestamps earlier --- api/kernel/os.hpp | 1 + src/kernel/os.cpp | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/api/kernel/os.hpp b/api/kernel/os.hpp index 4360fa6c60..4ac23afda4 100644 --- a/api/kernel/os.hpp +++ b/api/kernel/os.hpp @@ -277,6 +277,7 @@ class OS { static bool m_is_live_updated; static bool m_block_drivers_ready; static bool m_timestamps; + static bool m_timestamps_ready; static util::KHz cpu_khz_; static uintptr_t liveupdate_loc_; diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 35abc8f5c7..c8913905d4 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -55,6 +55,7 @@ bool OS::boot_sequence_passed_ = false; bool OS::m_is_live_updated = false; bool OS::m_block_drivers_ready = false; bool OS::m_timestamps = false; +bool OS::m_timestamps_ready = false; KHz OS::cpu_khz_ {-1}; uintptr_t OS::liveupdate_loc_ = 0; @@ -121,6 +122,9 @@ void OS::shutdown() void OS::post_start() { + // Enable timestamps (if present) + OS::m_timestamps_ready = true; + // LiveUpdate needs some initialization, although only if present OS::setup_liveupdate(); @@ -192,7 +196,7 @@ void OS::print(const char* str, const size_t len) } /** TIMESTAMPING **/ - if (OS::m_timestamps && OS::is_booted() && !OS::is_panicking()) + if (OS::m_timestamps && OS::m_timestamps_ready && !OS::is_panicking()) { static bool apply_ts = true; if (apply_ts) From 421f866c0c49fea29339b9ce74d0ba57632c2d5d Mon Sep 17 00:00:00 2001 From: John Zhang Date: Sat, 1 Sep 2018 00:36:23 +0800 Subject: [PATCH 574/723] clang-tidy performance-faster-string-find --- src/kernel/terminal.cpp | 6 +++--- src/plugins/unik.cpp | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/kernel/terminal.cpp b/src/kernel/terminal.cpp index 38d9018012..174ae6c7f6 100644 --- a/src/kernel/terminal.cpp +++ b/src/kernel/terminal.cpp @@ -132,7 +132,7 @@ split(const std::string& text, std::string& command) if (text.empty()) return retv; // extract command { - x = text.find(" "); + x = text.find(' '); // early return for cmd-only msg if (x == std::string::npos) { @@ -145,8 +145,8 @@ split(const std::string& text, std::string& command) } // parse remainder do { - x = text.find(" ", p+1); - size_t y = text.find(":", x+1); // find last param + x = text.find(' ', p+1); + size_t y = text.find(':', x+1); // find last param if (y == x+1) { // single argument diff --git a/src/plugins/unik.cpp b/src/plugins/unik.cpp index 5695c7bce0..8abab96eec 100644 --- a/src/plugins/unik.cpp +++ b/src/plugins/unik.cpp @@ -48,7 +48,7 @@ void unik::Client::register_instance(net::Inet& inet, const net::UDP::port_t por std::string strdata(data, len); INFO("Unik client","received UDP data from %s:%i: %s ", addr.to_string().c_str(), port, strdata.c_str()); - auto dotloc = strdata.find(":"); + auto dotloc = strdata.find(':'); if (dotloc == std::string::npos) { INFO("Unik client","Unexpected UDP data format - no ':' in string."); From 09619fd87db5a947543cf004322b1826ab8e4aea Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 3 Sep 2018 14:00:00 +0200 Subject: [PATCH 575/723] Add back full SMP support on x86_64 (no TLS) --- src/arch/x86_64/ist.cpp | 16 ++++++++--- src/kernel/events.cpp | 17 +++++++----- src/kernel/os.cpp | 8 +++--- src/kernel/syscalls.cpp | 12 +++++---- src/kernel/timers.cpp | 7 +++-- src/musl/writev.cpp | 10 +++---- src/platform/x86_pc/apic_revenant.cpp | 23 +++++++++------- src/platform/x86_pc/apic_timer.cpp | 10 ++++--- src/platform/x86_pc/idt.cpp | 2 +- src/platform/x86_pc/os.cpp | 2 +- src/platform/x86_pc/smp.cpp | 31 +++++++++++++++++++--- test/kernel/integration/smp/CMakeLists.txt | 3 ++- test/kernel/integration/smp/service.cpp | 2 +- 13 files changed, 96 insertions(+), 47 deletions(-) diff --git a/src/arch/x86_64/ist.cpp b/src/arch/x86_64/ist.cpp index 191ebb544b..1d047d4473 100644 --- a/src/arch/x86_64/ist.cpp +++ b/src/arch/x86_64/ist.cpp @@ -29,7 +29,7 @@ struct stack { void* phys; }; -static stack create_stack(size_t size, const char* name) +static stack create_stack_virt(size_t size, const char* name) { using namespace os; using namespace util::bitops; @@ -61,12 +61,19 @@ static stack create_stack(size_t size, const char* name) // Align stack pointer to bottom of stack minus a pop auto sp = map.lin + size - 8; - sp = sp & ~uintptr_t(0xf); + sp &= ~uintptr_t(0xf); // Force page fault if mapped area isn't writable ((char*)sp)[0] = '!'; - return {(void*)sp, phys}; + return {(void*) sp, phys}; +} +static stack create_stack_simple(size_t size, const char* /*name*/) +{ + auto* phys = (char*)memalign(4096, size); + uintptr_t sp = (uintptr_t) phys + size - 8; + sp &= ~uintptr_t(0xf); + return {(void*) sp, phys}; } namespace x86 @@ -83,6 +90,9 @@ namespace x86 void ist_initialize_for_cpu(int cpu, uintptr_t stack) { + typedef struct stack (*create_stack_func_t) (size_t, const char*); + create_stack_func_t create_stack = create_stack_virt; + if (cpu > 0) create_stack = create_stack_simple; auto& ist = lm_ist.at(cpu); memset(&ist.tss, 0, sizeof(AMD64_TSS)); diff --git a/src/kernel/events.cpp b/src/kernel/events.cpp index 31de5f0c71..93a8432394 100644 --- a/src/kernel/events.cpp +++ b/src/kernel/events.cpp @@ -42,14 +42,18 @@ void Events::init_local() { std::memset(event_subs.data(), 0, sizeof(event_subs)); std::memset(event_pend.data(), 0, sizeof(event_pend)); - // prevent legacy IRQs from being free for taking - for (int evt = 0; evt < 32; evt++) - event_subs[evt] = true; + + if (SMP::cpu_id() == 0) + { + // prevent legacy IRQs from being free for taking + for (int evt = 0; evt < 32; evt++) + event_subs[evt] = true; + } } uint8_t Events::subscribe(event_callback func) { - for (int evt = 32; evt < NUM_EVENTS; evt++) { + for (int evt = 0; evt < NUM_EVENTS; evt++) { if (event_subs[evt] == false) { subscribe(evt, func); return evt; @@ -59,9 +63,6 @@ uint8_t Events::subscribe(event_callback func) } void Events::subscribe(uint8_t evt, event_callback func) { - // enable IRQ in hardware - __arch_subscribe_irq(evt); - // Mark as subscribed to event_subs[evt] = true; // Set (new) callback for event @@ -77,6 +78,8 @@ void Events::subscribe(uint8_t evt, event_callback func) IRQ_BASE + evt, evt, SMP::cpu_id()); SMP::global_unlock(); #endif + // enable IRQ in hardware + __arch_subscribe_irq(evt); } } void Events::unsubscribe(uint8_t evt) diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index c8913905d4..05df07d7b1 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -202,7 +202,7 @@ void OS::print(const char* str, const size_t len) if (apply_ts) { std::string ts = "[" + isotime::now() + "] "; - for (auto& callback : os_print_handlers) { + for (const auto& callback : os_print_handlers) { callback(ts.c_str(), ts.size()); } apply_ts = false; @@ -212,9 +212,9 @@ void OS::print(const char* str, const size_t len) } /** TIMESTAMPING **/ - for (auto& callback : os_print_handlers) { - if (os_enable_boot_logging || OS::is_booted() || OS::is_panicking()) - { + if (os_enable_boot_logging || OS::is_booted() || OS::is_panicking()) + { + for (const auto& callback : os_print_handlers) { callback(str, len); } } diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index 44667b68e4..9c3211faa1 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -66,9 +66,11 @@ void syscall_SYS_exit_group(int status) struct alignas(SMP_ALIGN) context_buffer { std::array buffer; - int panics = 0; }; static SMP::Array contexts; +// NOTE: panics cannot be per-cpu because it might not be ready yet +// NOTE: it's also used by OS::is_panicking(), used by OS::print(...) +static int panics = 0; size_t get_crash_context_length() { @@ -80,12 +82,13 @@ char* get_crash_context_buffer() } bool OS::is_panicking() noexcept { - return PER_CPU(contexts).panics > 0; + return panics > 0; } extern "C" void cpu_enable_panicking() { - PER_CPU(contexts).panics++; + //PER_CPU(contexts).panics++; + __sync_fetch_and_add(&panics, 1); } static OS::on_panic_func panic_handler = nullptr; @@ -111,8 +114,7 @@ void panic_perform_inspection_procedure() {} void panic(const char* why) { cpu_enable_panicking(); - if (PER_CPU(contexts).panics > 4) - double_fault(why); + if (panics > 4) double_fault(why); const int current_cpu = SMP::cpu_id(); diff --git a/src/kernel/timers.cpp b/src/kernel/timers.cpp index 220f2a97bd..1bb6cf7cb9 100644 --- a/src/kernel/timers.cpp +++ b/src/kernel/timers.cpp @@ -121,8 +121,11 @@ void Timers::ready() if (get().is_running == false) { timers_handler(); } - // call Service::ready(), because timer system is ready! - Service::ready(); + if (SMP::cpu_id() == 0) + { + // call Service::ready(), because timer system is ready! + Service::ready(); + } } Timers::id_t Timers::periodic(duration_t when, duration_t period, handler_t handler) diff --git a/src/musl/writev.cpp b/src/musl/writev.cpp index c9ffb77b50..225b0d7eb8 100644 --- a/src/musl/writev.cpp +++ b/src/musl/writev.cpp @@ -1,13 +1,11 @@ #include "common.hpp" - -#include #include -static ssize_t sys_writev(int fd, const struct iovec *iov, int iovcnt) +static long sys_writev(int fd, const struct iovec *iov, int iovcnt) { - if(fd <= 2) + if (fd == 1 || fd == 2) { - ssize_t res = 0; + long res = 0; for(int i = 0; i < iovcnt; i++) { auto* text = (const char*)iov[i].iov_base; @@ -21,7 +19,7 @@ static ssize_t sys_writev(int fd, const struct iovec *iov, int iovcnt) } extern "C" -ssize_t syscall_SYS_writev(int fd, const struct iovec *iov, int iovcnt){ +long syscall_SYS_writev(int fd, const struct iovec *iov, int iovcnt){ //return strace(sys_writev, "writev", fd, iov, iovcnt); return sys_writev(fd, iov, iovcnt); } diff --git a/src/platform/x86_pc/apic_revenant.cpp b/src/platform/x86_pc/apic_revenant.cpp index 406fdcb6e2..62694d4ce6 100644 --- a/src/platform/x86_pc/apic_revenant.cpp +++ b/src/platform/x86_pc/apic_revenant.cpp @@ -79,30 +79,33 @@ void revenant_main(int cpu) // enable Local APIC x86::APIC::get().smp_enable(); // setup GDT & per-cpu feature - initialize_cpu_tables_for_cpu(cpu); -#ifdef ARCH_x86_64 - // interrupt stack tables - ist_initialize_for_cpu(cpu, this_stack); -#endif + x86::initialize_cpu_tables_for_cpu(cpu); // show we are online, and verify CPU ID is correct SMP::global_lock(); auto stack = (uintptr_t) get_cpu_esp(); INFO2("AP %d started at %p", SMP::cpu_id(), (void*) this_stack); SMP::global_unlock(); + // initialize exceptions before asserts + x86::idt_initialize_for_cpu(cpu); assert(cpu == SMP::cpu_id()); assert(stack >= this_stack_end && stack < this_stack); - x86::idt_initialize_for_cpu(cpu); - Events::get(cpu).init_local(); +#ifdef ARCH_x86_64 + // interrupt stack tables + ist_initialize_for_cpu(cpu, this_stack); +#endif + + auto& ev = Events::get(cpu); + ev.init_local(); + // subscribe to task and timer interrupts + ev.subscribe(0, revenant_task_handler); + ev.subscribe(1, APIC_Timer::start_timers); // enable interrupts asm volatile("sti"); // init timer system APIC_Timer::init(); // initialize clocks Clocks::init(); - // subscribe to task and timer interrupts - Events::get().subscribe(0, revenant_task_handler); - Events::get().subscribe(1, APIC_Timer::start_timers); // seed RNG RNG::get().init(); diff --git a/src/platform/x86_pc/apic_timer.cpp b/src/platform/x86_pc/apic_timer.cpp index bf3a1e48e5..c905ac7c55 100644 --- a/src/platform/x86_pc/apic_timer.cpp +++ b/src/platform/x86_pc/apic_timer.cpp @@ -45,7 +45,7 @@ namespace x86 int intr; bool intr_enabled = false; }; - static std::array timerdata; + static SMP::Array timerdata; #define GET_TIMER() PER_CPU(timerdata) @@ -57,10 +57,10 @@ namespace x86 stop); // timer stop function // set interrupt handler - PER_CPU(timerdata).intr = + GET_TIMER().intr = Events::get().subscribe(Timers::timers_handler); // initialize local APIC timer - APIC::get().timer_init(PER_CPU(timerdata).intr); + APIC::get().timer_init(GET_TIMER().intr); } void APIC_Timer::calibrate() { @@ -68,6 +68,10 @@ namespace x86 if (ticks_per_micro != 0) { start_timers(); + // with SMP, signal everyone else too (IRQ 1) + if (SMP::cpu_count() > 1) { + APIC::get().bcast_ipi(0x21); + } return; } diff --git a/src/platform/x86_pc/idt.cpp b/src/platform/x86_pc/idt.cpp index 996013c196..827efea8cc 100644 --- a/src/platform/x86_pc/idt.cpp +++ b/src/platform/x86_pc/idt.cpp @@ -120,7 +120,7 @@ extern "C" { void x86_IDT::init() { // make sure its all zeroes - memset(&PER_CPU(idt).entry, 0, sizeof(x86_IDT::entry)); + memset(&this->entry[0], 0, sizeof(x86_IDT::entry)); set_exception_handler(0, __cpu_except_0); set_exception_handler(1, __cpu_except_1); diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index ceb38a6dd6..e7369b6776 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -15,7 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -//#define DEBUG +#define DEBUG #define MYINFO(X,...) INFO("Kernel", X, ##__VA_ARGS__) #include diff --git a/src/platform/x86_pc/smp.cpp b/src/platform/x86_pc/smp.cpp index f5fc5f659f..d014c2c5b1 100644 --- a/src/platform/x86_pc/smp.cpp +++ b/src/platform/x86_pc/smp.cpp @@ -45,6 +45,26 @@ struct apic_boot { uint32_t stack_size; }; +struct __libc { + int can_do_threads; + int threaded; + int secure; + volatile int threads_minus_1; + size_t* auxv; +}; +extern struct __libc __libc; +//extern "C" struct __libc *__libc_loc(void) __attribute__((const)); +//#define __libc (*__libc_loc()) + +static inline void musl_override_glob_locks() +{ + printf("__libc.can_do_threads: %d __libc.threaded: %d\n", + __libc.can_do_threads, __libc.threaded); + printf("__libc.threads_minus_1: %d -> %d\n", + __libc.threads_minus_1, 1); + __libc.threads_minus_1 = 1; +} + namespace x86 { @@ -53,6 +73,8 @@ void init_SMP() const uint32_t CPUcount = ACPI::get_cpus().size(); if (CPUcount <= 1) return; assert(CPUcount <= SMP_MAX_CORES); + // avoid heap usage during AP init + x86::smp_main.initialized_cpus.reserve(CPUcount); // copy our bootloader to APIC init location const char* start = &_binary_apic_boot_bin_start; @@ -85,6 +107,9 @@ void init_SMP() // reset barrier smp_main.boot_barrier.reset(1); + // enable global locks on musl + musl_override_glob_locks(); + auto& apic = x86::APIC::get(); // turn on CPUs INFO("SMP", "Initializing APs"); @@ -95,7 +120,7 @@ void init_SMP() cpu.cpu, cpu.id, cpu.flags); apic.ap_init(cpu.id); } - PIT::blocking_cycles(10); + //PIT::blocking_cycles(10); // start CPUs INFO("SMP", "Starting APs"); @@ -104,9 +129,9 @@ void init_SMP() if (cpu.id == apic.get_id()) continue; // Send SIPI with start page at BOOTLOADER_LOCATION apic.ap_start(cpu.id, BOOTLOADER_LOCATION >> 12); - //apic.ap_start(cpu.id, BOOTLOADER_LOCATION >> 12); + apic.ap_start(cpu.id, BOOTLOADER_LOCATION >> 12); } - PIT::blocking_cycles(1); + //PIT::blocking_cycles(1); // wait for all APs to start smp_main.boot_barrier.spin_wait(CPUcount); diff --git a/test/kernel/integration/smp/CMakeLists.txt b/test/kernel/integration/smp/CMakeLists.txt index bbc8363eaf..95974a0e6c 100644 --- a/test/kernel/integration/smp/CMakeLists.txt +++ b/test/kernel/integration/smp/CMakeLists.txt @@ -1,4 +1,5 @@ cmake_minimum_required(VERSION 2.8.9) +option(threading "" ON) # IncludeOS install location if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) @@ -9,7 +10,6 @@ project (service) # Human-readable name of your service set(SERVICE_NAME "SMP Test") - # Name of your service binary set(BINARY "smp") @@ -19,6 +19,7 @@ set(SOURCES ) set(DRIVERS + boot_logger ) set(PLUGINS diff --git a/test/kernel/integration/smp/service.cpp b/test/kernel/integration/smp/service.cpp index 0e33319224..e2e6f1dac7 100644 --- a/test/kernel/integration/smp/service.cpp +++ b/test/kernel/integration/smp/service.cpp @@ -89,7 +89,7 @@ void smp_advanced_test() } SMP::global_unlock(); }); - }); + }, i); // start working on tasks SMP::signal(); } From 695cba7a1af4a895c682e642eafd016e9a2e929b Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 3 Sep 2018 14:04:49 +0200 Subject: [PATCH 576/723] test: Disable actual SMP since Jenkins don't build with it --- test/kernel/integration/smp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/kernel/integration/smp/CMakeLists.txt b/test/kernel/integration/smp/CMakeLists.txt index 95974a0e6c..212f89a229 100644 --- a/test/kernel/integration/smp/CMakeLists.txt +++ b/test/kernel/integration/smp/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 2.8.9) -option(threading "" ON) +#option(threading "" ON) # IncludeOS install location if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) From c3ff7b59bb134400b933a09f0b645934bf50f2b4 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 3 Sep 2018 16:50:42 +0200 Subject: [PATCH 577/723] cmake: Remove old defines, rename threading to smp --- CMakeLists.txt | 12 +++--------- cmake/post.service.cmake | 7 +++---- cmake/pre.service.cmake | 2 +- src/CMakeLists.txt | 2 +- test/kernel/integration/smp/CMakeLists.txt | 2 +- 5 files changed, 9 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 693e6c8c60..1b719bd40d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,7 +48,7 @@ else() set(DEFAULT_VM "vm.cpu_feat.json") # vmrunner endif(cpu_feat_vanilla) -option(threading "Compile with SMP/threading support" OFF) +option(smp "Compile with SMP (multiprocessing)" OFF) option(silent "Disable most output during OS boot" OFF) @@ -137,20 +137,14 @@ endif() enable_language(ASM_NASM) - -if (NOT threading) - add_definitions(-D_LIBCPP_HAS_NO_THREADS) -endif() - - # initialize C and C++ compiler flags if (CMAKE_COMPILER_IS_GNUCC) # gcc/g++ settings - set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c ${LIBCPP_THREADING}") + set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c") set(CMAKE_C_FLAGS " -MMD ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c") else() # these kinda work with llvm - set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c ${LIBCPP_THREADING}") + set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c") set(CMAKE_C_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c") endif() diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 3dc17e45ad..91dac7af6d 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -28,12 +28,11 @@ else() endif() enable_language(ASM_NASM) -if (NOT threading) +if (NOT smp) add_definitions(-DINCLUDEOS_SINGLE_THREADED) - add_definitions(-D_LIBCPP_HAS_NO_THREADS) - message(STATUS "Building without threading / SMP") + message(STATUS "Building without SMP") else() - message(STATUS "Building with threading / SMP") + message(STATUS "Building with SMP") endif() if (coroutines) diff --git a/cmake/pre.service.cmake b/cmake/pre.service.cmake index c92cce6fcf..f9388166d8 100644 --- a/cmake/pre.service.cmake +++ b/cmake/pre.service.cmake @@ -22,7 +22,7 @@ option(debug "Build with debugging symbols (OBS: increases binary size)" OFF) option(minimal "Build for minimal size" OFF) option(stripped "Strip symbols to further reduce size" OFF) -option(threading "Compile threading and SMP support" OFF) +option(smp "Enable SMP (multiprocessing)" OFF) option(undefined_san "Enable undefined-behavior sanitizer" OFF) option(thin_lto "Enable Thin LTO plugin" OFF) option(full_lto "Enable full LTO (also works on LD)" OFF) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 788602526d..dc137b5066 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,7 +3,7 @@ # add_definitions(-DARCH_${ARCH}) add_definitions(-DARCH="${ARCH}") -if (NOT threading) +if (NOT smp) add_definitions(-DINCLUDEOS_SINGLE_THREADED) endif() diff --git a/test/kernel/integration/smp/CMakeLists.txt b/test/kernel/integration/smp/CMakeLists.txt index 212f89a229..0af7c9721c 100644 --- a/test/kernel/integration/smp/CMakeLists.txt +++ b/test/kernel/integration/smp/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 2.8.9) -#option(threading "" ON) +#option(smp "" ON) # IncludeOS install location if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) From c37c3dc36eb218c9c142fea7d7921ab263b757a0 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 4 Sep 2018 09:43:14 +0200 Subject: [PATCH 578/723] cmake: Remove SINGLE_THREADED, add SMP_ENABLE option --- api/common | 6 +-- api/kernel/fiber.hpp | 18 ++++----- api/net/buffer_store.hpp | 4 +- api/smp | 44 ++++++++++----------- api/smp_utils | 8 ++-- api/util/statman.hpp | 2 +- cmake/post.service.cmake | 8 ++-- linux/CMakeLists.txt | 1 - src/CMakeLists.txt | 4 +- src/kernel/events.cpp | 2 +- src/kernel/fiber.cpp | 6 +-- src/net/buffer_store.cpp | 4 +- src/platform/x86_pc/platform.cpp | 2 +- src/platform/x86_pc/smp.cpp | 24 +++++------ src/util/statman.cpp | 2 +- test/CMakeLists.txt | 1 - test/kernel/integration/fiber/service.cpp | 8 +--- test/stl/integration/coroutines/service.cpp | 18 ++++----- 18 files changed, 73 insertions(+), 89 deletions(-) diff --git a/api/common b/api/common index 2dc452b642..cc0aeb144d 100644 --- a/api/common +++ b/api/common @@ -53,18 +53,18 @@ static_assert(sizeof(void*) == 8, "Pointer must match arch"); #undef Expects #undef Ensures -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE #include #endif #include inline void __expect_fail(const char *expr, const char *file, int line, const char *func){ -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE SMP::global_lock(); #endif fprintf(stderr, "%s:%i:%s %s \n",file, line, func, expr); fflush(NULL); -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE SMP::global_unlock(); #endif panic(expr); diff --git a/api/kernel/fiber.hpp b/api/kernel/fiber.hpp index 16f3608d84..5f13d5071a 100644 --- a/api/kernel/fiber.hpp +++ b/api/kernel/fiber.hpp @@ -23,7 +23,7 @@ #include #include -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE #include #endif @@ -207,21 +207,19 @@ class Fiber { static int last_id() { -#if defined( INCLUDEOS_SINGLE_THREADED) - return next_id_; -#else +#ifdef INCLUDEOS_SMP_ENABLE return next_id_.load(); +#else + return next_id_; #endif } private: - - -#if defined(INCLUDEOS_SINGLE_THREADED) - static int next_id_; -#else +#ifdef INCLUDEOS_SMP_ENABLE static std::atomic next_id_; +#else + static int next_id_; #endif static SMP::Array main_; static SMP::Array current_; @@ -255,8 +253,6 @@ class Fiber { bool running_ { false }; friend void ::fiber_jumpstarter(Fiber* f); - }; - #endif diff --git a/api/net/buffer_store.hpp b/api/net/buffer_store.hpp index e5e8e7b73d..50364a7d27 100644 --- a/api/net/buffer_store.hpp +++ b/api/net/buffer_store.hpp @@ -95,7 +95,7 @@ namespace net std::vector available_; std::unique_ptr next_; int index; -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE // has strict alignment reqs, so put at end spinlock_t plock = 0; #endif @@ -110,7 +110,7 @@ namespace net auto* buff = (uint8_t*) addr; // try to release directly into pool if (LIKELY(is_from_this_pool(buff))) { -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE scoped_spinlock spinlock(this->plock); #endif available_.push_back(buff); diff --git a/api/smp b/api/smp index 5722ed5426..7fb3f2b836 100644 --- a/api/smp +++ b/api/smp @@ -25,10 +25,10 @@ #include #ifndef SMP_MAX_CORES -# ifdef INCLUDEOS_SINGLE_THREADED -# define SMP_MAX_CORES 1 -# else +# ifdef INCLUDEOS_SMP_ENABLE # define SMP_MAX_CORES 32 +# else +# define SMP_MAX_CORES 1 # endif #endif @@ -102,26 +102,26 @@ public: #endif #include -#ifdef INCLUDEOS_SINGLE_THREADED -template -inline T& per_cpu_help(std::array& array) -{ - return array[0]; -} -#else -template -inline T& per_cpu_help(std::array& array) -{ - unsigned cpuid; -#ifdef ARCH_x86_64 - asm("movl %%gs:(0x0), %0" : "=r" (cpuid)); -#elif defined(ARCH_i686) - asm("movl %%fs:(0x0), %0" : "=r" (cpuid)); +#ifdef INCLUDEOS_SMP_ENABLE + template + inline T& per_cpu_help(std::array& array) + { + unsigned cpuid; +# ifdef ARCH_x86_64 + asm("movl %%gs:(0x0), %0" : "=r" (cpuid)); +# elif defined(ARCH_i686) + asm("movl %%fs:(0x0), %0" : "=r" (cpuid)); +# else + #error "Implement me?" +# endif + return array.at(cpuid); + } #else - #error "Implement me?" -#endif - return array.at(cpuid); -} + template + inline T& per_cpu_help(std::array& array) + { + return array[0]; + } #endif #endif diff --git a/api/smp_utils b/api/smp_utils index eabf9736ce..88c8bf5f28 100644 --- a/api/smp_utils +++ b/api/smp_utils @@ -27,10 +27,7 @@ // Intel 3a 8.10.6.7: 128-byte boundary typedef unsigned int spinlock_t __attribute__((aligned(128))); -#ifdef INCLUDEOS_SINGLE_THREADED -inline void lock(spinlock_t&) {} -inline void unlock(spinlock_t&) {} -#else +#ifdef INCLUDEOS_SMP_ENABLE inline void lock(spinlock_t& lock) { while (!__sync_bool_compare_and_swap(&lock, 0, 1)) { while (lock) asm("pause"); @@ -39,6 +36,9 @@ inline void lock(spinlock_t& lock) { inline void unlock(spinlock_t& lock) { __sync_lock_release(&lock, 0); // barrier } +#else +inline void lock(spinlock_t&) {} +inline void unlock(spinlock_t&) {} #endif struct scoped_spinlock diff --git a/api/util/statman.hpp b/api/util/statman.hpp index 6f162e4dfe..a60aa1ef54 100644 --- a/api/util/statman.hpp +++ b/api/util/statman.hpp @@ -192,7 +192,7 @@ class Statman { Stat* end_stats_; MemBitmap::word* bdata = nullptr; MemBitmap bitmap; -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE spinlock_t stlock = 0; #endif diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 91dac7af6d..202dee242c 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -28,11 +28,9 @@ else() endif() enable_language(ASM_NASM) -if (NOT smp) - add_definitions(-DINCLUDEOS_SINGLE_THREADED) - message(STATUS "Building without SMP") -else() - message(STATUS "Building with SMP") +if (smp) + add_definitions(-DINCLUDEOS_SMP_ENABLE) + message(STATUS "Building with SMP enabled") endif() if (coroutines) diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index 7226c88ebb..231a2130f1 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -44,7 +44,6 @@ add_definitions(-DOS_VERSION=\"${OS_VERSION}\") add_definitions(-DOS_TERMINATE_ON_CONTRACT_VIOLATION) add_definitions(-DARP_PASSTHROUGH) add_definitions(-DNO_DEBUG) -add_definitions(-DINCLUDEOS_SINGLE_THREADED) add_definitions(-DUSERSPACE_LINUX) include_directories(../api) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dc137b5066..2cb929e69f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -3,8 +3,8 @@ # add_definitions(-DARCH_${ARCH}) add_definitions(-DARCH="${ARCH}") -if (NOT smp) -add_definitions(-DINCLUDEOS_SINGLE_THREADED) +if (smp) +add_definitions(-DINCLUDEOS_SMP_ENABLE) endif() include_directories(${LIBCXX_INCLUDE_DIR}) diff --git a/src/kernel/events.cpp b/src/kernel/events.cpp index 93a8432394..4f39f68fb0 100644 --- a/src/kernel/events.cpp +++ b/src/kernel/events.cpp @@ -26,7 +26,7 @@ static SMP::Array managers; Events& Events::get(int cpuid) { -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE return managers.at(cpuid); #else (void) cpuid; diff --git a/src/kernel/fiber.cpp b/src/kernel/fiber.cpp index 069088714d..bae0b77049 100644 --- a/src/kernel/fiber.cpp +++ b/src/kernel/fiber.cpp @@ -23,10 +23,10 @@ #include // Default location for previous stack. Asm will always save a pointer. -#if defined(INCLUDEOS_SINGLE_THREADED) -int Fiber::next_id_{0}; -#else +#ifdef INCLUDEOS_SMP_ENABLE std::atomic Fiber::next_id_{0}; +#else +int Fiber::next_id_{0}; #endif SMP::Array Fiber::main_ = {{nullptr}}; diff --git a/src/net/buffer_store.cpp b/src/net/buffer_store.cpp index 9a855a8309..94a6a97bb8 100644 --- a/src/net/buffer_store.cpp +++ b/src/net/buffer_store.cpp @@ -126,7 +126,7 @@ namespace net { BufferStore::buffer_t BufferStore::get_buffer() { -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE scoped_spinlock spinlock(plock); #endif @@ -157,7 +157,7 @@ namespace net { auto* buff = (uint8_t*) addr; BSD_RELEASE("%d: Release %p -> ", this->index, buff); -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE scoped_spinlock spinlock(plock); #endif #ifdef ENABLE_BUFFERSTORE_CHAIN diff --git a/src/platform/x86_pc/platform.cpp b/src/platform/x86_pc/platform.cpp index 880a429f41..c2d27fe5fd 100644 --- a/src/platform/x86_pc/platform.cpp +++ b/src/platform/x86_pc/platform.cpp @@ -75,7 +75,7 @@ void __platform_init() asm volatile("sti"); // initialize and start registered APs found in ACPI-tables -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE x86::init_SMP(); #endif diff --git a/src/platform/x86_pc/smp.cpp b/src/platform/x86_pc/smp.cpp index d014c2c5b1..ec6e15a382 100644 --- a/src/platform/x86_pc/smp.cpp +++ b/src/platform/x86_pc/smp.cpp @@ -194,31 +194,29 @@ void SMP::init_task() void SMP::add_task(smp_task_func task, smp_done_func done, int cpu) { -#ifdef INCLUDEOS_SINGLE_THREADED - assert(cpu == 0); - task(); done(); -#else +#ifdef INCLUDEOS_SMP_ENABLE lock(smp_system[cpu].tlock); smp_system[cpu].tasks.emplace_back(std::move(task), std::move(done)); unlock(smp_system[cpu].tlock); +#else + assert(cpu == 0); + task(); done(); #endif } void SMP::add_task(smp_task_func task, int cpu) { -#ifdef INCLUDEOS_SINGLE_THREADED - assert(cpu == 0); - task(); -#else +#ifdef INCLUDEOS_SMP_ENABLE lock(smp_system[cpu].tlock); smp_system[cpu].tasks.emplace_back(std::move(task), nullptr); unlock(smp_system[cpu].tlock); +#else + assert(cpu == 0); + task(); #endif } void SMP::add_bsp_task(smp_done_func task) { -#ifdef INCLUDEOS_SINGLE_THREADED - task(); -#else +#ifdef INCLUDEOS_SMP_ENABLE // queue job auto& system = PER_CPU(smp_system); lock(system.flock); @@ -228,12 +226,14 @@ void SMP::add_bsp_task(smp_done_func task) smp_main.bitmap.atomic_set(SMP::cpu_id()); // call home x86::APIC::get().send_bsp_intr(); +#else + task(); #endif } void SMP::signal(int cpu) { -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE // broadcast that there is work to do // 0: Broadcast to everyone except BSP if (cpu == 0) diff --git a/src/util/statman.cpp b/src/util/statman.cpp index ad4b761a2d..4cfe3b63ee 100644 --- a/src/util/statman.cpp +++ b/src/util/statman.cpp @@ -101,7 +101,7 @@ Statman::~Statman() } Stat& Statman::create(const Stat::Stat_type type, const std::string& name) { -#ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE volatile scoped_spinlock lock(this->stlock); #endif if (name.empty()) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index fc2d85bcfd..ed4ad14396 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -25,7 +25,6 @@ add_definitions(-DARCH_${ARCH}) add_definitions(-DARCH="${ARCH}") add_definitions(-DPLATFORM_UNITTEST) add_definitions(-DOS_VERSION="v0.0.0.1") -add_definitions(-DINCLUDEOS_SINGLE_THREADED) set(CMAKE_C_FLAGS "-g -O0 -std=c11 -Wall -Wextra") diff --git a/test/kernel/integration/fiber/service.cpp b/test/kernel/integration/fiber/service.cpp index e8ce2f7993..b8f15555d3 100644 --- a/test/kernel/integration/fiber/service.cpp +++ b/test/kernel/integration/fiber/service.cpp @@ -311,19 +311,15 @@ void Service::start() INFO("Service", "Computed long: %li", ret); - #ifndef INCLUDEOS_SINGLE_THREADED +#ifdef INCLUDEOS_SMP_ENABLE if (SMP::cpu_count() > 1) { extern void fiber_smp_test(); fiber_smp_test(); } else { INFO("Service", "SMP test requires > 1 cpu's, found %i \n", SMP::cpu_count()); } - #endif - +#endif SMP_PRINT("Service done. rsp @ %p \n", get_rsp()); - SMP_PRINT("SUCCESS\n"); exit(0); - - } diff --git a/test/stl/integration/coroutines/service.cpp b/test/stl/integration/coroutines/service.cpp index 7dd100b36a..fed2ab4978 100644 --- a/test/stl/integration/coroutines/service.cpp +++ b/test/stl/integration/coroutines/service.cpp @@ -34,10 +34,10 @@ struct smp_future { using handle_type = std::experimental::coroutine_handle; handle_type coro; -#if defined(INCLUDEOS_SINGLE_THREADED) - int done {false}; -#else +#ifdef INCLUDEOS_SMP_ENABLE std::atomic done {false}; +#else + int done {false}; #endif smp_future(const smp_future &s) = delete; @@ -151,26 +151,22 @@ smp_future reduce() { int cpu = 1; for (auto& i : futures) { -#if defined(INCLUDEOS_SINGLE_THREADED) - i.coro(); - i.done = true; -#else +#ifdef INCLUDEOS_SMP_ENABLE SMP::add_task([&i]() { CPULOG("Resuming coroutine \n"); i.coro(); CPULOG("Coroutine done. \n"); i.done.store(true); }, []{}, cpu); +#else + i.coro(); + i.done = true; #endif cpu++; } - SMP::signal(); - CPULOG("Created %li coroutines \n", futures.size()); - - asm("pause"); int sum = 0; From 43ec1bb86cc71f562f739aa522601d2f2a8fc1de Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Tue, 4 Sep 2018 11:56:39 +0200 Subject: [PATCH 579/723] NaCl: Syslog transpilation update (ip().str() -> ip().to_string()) --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index 5bc2ab0634..1c801d0203 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit 5bc2ab0634735177607f68af023b99800f9fdaa2 +Subproject commit 1c801d0203850fbc8eb52faeabcdfb48e17218f0 From 405b1fa7438824bab49c9fe5d90a3f3fcf5cbc08 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 4 Sep 2018 12:54:35 +0200 Subject: [PATCH 580/723] vmxnet3: Use deq for sendq, add stat --- src/drivers/vmxnet3.cpp | 29 +++++++++++++++++------------ src/drivers/vmxnet3.hpp | 7 +++++-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index df65790ba6..df1d30e8ce 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -121,7 +122,9 @@ static inline uint16_t buffer_size_for_mtu(const uint16_t mtu) vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : Link(Link_protocol{{this, &vmxnet3::transmit}, mac()}), - m_pcidev(d), m_mtu(mtu), bufstore_{1024, buffer_size_for_mtu(mtu)} + m_pcidev(d), m_mtu(mtu), + sendq_stat{Statman::get().create(Stat::UINT32, device_name() + ".sendq").get_uint32()}, + bufstore_{1024, buffer_size_for_mtu(mtu)} { INFO("vmxnet3", "Driver initializing (rev=%#x)", d.rev_id()); assert(d.rev_id() == REVISION_ID); @@ -465,8 +468,8 @@ bool vmxnet3::transmit_handler() tx.buffers[desc] = nullptr; } // try to send sendq first - if (this->can_transmit() && sendq != nullptr) { - this->transmit(std::move(sendq)); + if (this->can_transmit() && !sendq.empty()) { + this->transmit(nullptr); transmitted = true; } // if we can still send more, message network stack @@ -515,20 +518,22 @@ bool vmxnet3::receive_handler(const int Q) void vmxnet3::transmit(net::Packet_ptr pckt_ptr) { - if (sendq == nullptr) - sendq = std::move(pckt_ptr); - else - sendq->chain(std::move(pckt_ptr)); + while (pckt_ptr != nullptr) { + auto tail = pckt_ptr->detach_tail(); + sendq.emplace_back(std::move(pckt_ptr)); + pckt_ptr = std::move(tail); + } // send as much as possible from sendq - while (sendq != nullptr && can_transmit()) + while (!sendq.empty() && can_transmit()) { - auto next = sendq->detach_tail(); + auto* packet = sendq.front().release(); + sendq.pop_front(); // transmit released buffer - auto* packet = sendq.release(); transmit_data(packet->buf() + DRIVER_OFFSET, packet->size()); - // next is the new sendq - sendq = std::move(next); } + // update stat + sendq_stat = sendq.size(); + // delay dma message until we have written as much as possible if (!deferred_kick) { diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index 7479674099..db7e7d838a 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include struct vmxnet3_dma; struct vmxnet3_rx_desc; @@ -138,7 +139,9 @@ class vmxnet3 : public net::Link_layer bool already_polling = false; bool link_state_up = false; static void handle_deferred(); - // sendq as packet chain - net::Packet_ptr sendq = nullptr; + + // sendq as double-ended q + uint32_t& sendq_stat; + std::deque sendq; net::BufferStore bufstore_; }; From 6383054bfe8f5d009c2e5a72e88113b158c87318 Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Tue, 4 Sep 2018 13:00:51 +0200 Subject: [PATCH 581/723] NaCl: Updating syslog goldenfile and comments --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index 1c801d0203..986837d21c 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit 1c801d0203850fbc8eb52faeabcdfb48e17218f0 +Subproject commit 986837d21ce2da280f18f5d2e52360025ed230f2 From 90286fbf6d632f0cce1f3756a0da096a76483850 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 4 Sep 2018 18:47:39 +0200 Subject: [PATCH 582/723] vmxnet3: Add sendq_cur and sendq_max, as well as buffer_size stats --- src/drivers/vmxnet3.cpp | 10 +++++++--- src/drivers/vmxnet3.hpp | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index df1d30e8ce..e4c7666140 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -123,11 +123,14 @@ static inline uint16_t buffer_size_for_mtu(const uint16_t mtu) vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : Link(Link_protocol{{this, &vmxnet3::transmit}, mac()}), m_pcidev(d), m_mtu(mtu), - sendq_stat{Statman::get().create(Stat::UINT32, device_name() + ".sendq").get_uint32()}, + stat_sendq_cur{Statman::get().create(Stat::UINT32, device_name() + ".sendq_now").get_uint32()}, + stat_sendq_max{Statman::get().create(Stat::UINT32, device_name() + ".sendq_max").get_uint32()}, bufstore_{1024, buffer_size_for_mtu(mtu)} { INFO("vmxnet3", "Driver initializing (rev=%#x)", d.rev_id()); assert(d.rev_id() == REVISION_ID); + Statman::get().create(Stat::UINT32, device_name() + ".buffer_size") + .get_uint32() = bufstore_.bufsize(); // find and store capabilities d.parse_capabilities(); @@ -531,8 +534,9 @@ void vmxnet3::transmit(net::Packet_ptr pckt_ptr) // transmit released buffer transmit_data(packet->buf() + DRIVER_OFFSET, packet->size()); } - // update stat - sendq_stat = sendq.size(); + // update sendq stats + stat_sendq_cur = sendq.size(); + stat_sendq_max = std::max(stat_sendq_max, stat_sendq_cur); // delay dma message until we have written as much as possible if (!deferred_kick) diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index db7e7d838a..2296452d72 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -141,7 +141,8 @@ class vmxnet3 : public net::Link_layer static void handle_deferred(); // sendq as double-ended q - uint32_t& sendq_stat; + uint32_t& stat_sendq_cur; + uint32_t& stat_sendq_max; std::deque sendq; net::BufferStore bufstore_; }; From bbaaa54661f868b03fa6aca41040be0608185eb5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 5 Sep 2018 14:23:56 +0200 Subject: [PATCH 583/723] x86: Don't use fs/gs without SMP --- src/musl/mmap.cpp | 1 - src/platform/x86_pc/smp.cpp | 11 +++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index 65ec29189b..ad27dd9fb4 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -20,7 +20,6 @@ uintptr_t __init_mmap(uintptr_t addr_begin) auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); int64_t len = (OS::heap_max() - aligned_begin) & ~int64_t(Alloc::align - 1); - kprintf("Initializing mmap\n"); alloc = Alloc::create((void*)aligned_begin, len); kprintf("* mmap initialized. Begin: 0x%zx, end: 0x%zx\n", addr_begin, addr_begin + len); diff --git a/src/platform/x86_pc/smp.cpp b/src/platform/x86_pc/smp.cpp index ec6e15a382..2d962933e0 100644 --- a/src/platform/x86_pc/smp.cpp +++ b/src/platform/x86_pc/smp.cpp @@ -167,6 +167,7 @@ using namespace x86; /// implementation of the SMP interface /// int SMP::cpu_id() noexcept { +#ifdef INCLUDEOS_SMP_ENABLE int cpuid; #ifdef ARCH_x86_64 asm("movl %%gs:(0x0), %0" : "=r" (cpuid)); @@ -176,13 +177,15 @@ int SMP::cpu_id() noexcept #error "Implement me?" #endif return cpuid; +#else + return 0; +#endif } -int SMP::cpu_count() noexcept -{ + +int SMP::cpu_count() noexcept { return x86::smp_main.initialized_cpus.size(); } - -const std::vector& SMP::active_cpus(){ +const std::vector& SMP::active_cpus() { return x86::smp_main.initialized_cpus; } From 747e836f71b057bd529fa799f6e627d59d6979e1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 5 Sep 2018 14:25:01 +0200 Subject: [PATCH 584/723] x86: Avoid doing sanity checks too early --- src/platform/x86_pc/kernel_start.cpp | 1 - src/platform/x86_pc/sanity_checks.cpp | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index e0c0d4fb50..137bfe03ec 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -241,7 +241,6 @@ void kernel_start(uint32_t magic, uint32_t addr) #warning Classical syscall interface missing for 32-bit #endif - kernel_sanity_checks(); // GDB_ENTRY; PRATTLE("* Starting libc initialization\n"); __libc_start_main(kernel_main, argc, argv.data()); diff --git a/src/platform/x86_pc/sanity_checks.cpp b/src/platform/x86_pc/sanity_checks.cpp index 85ee91342e..cfc4f12d9f 100644 --- a/src/platform/x86_pc/sanity_checks.cpp +++ b/src/platform/x86_pc/sanity_checks.cpp @@ -22,11 +22,11 @@ #include #include -//#define ENABLE_CRC_RO +#define ENABLE_CRC_RO // Global constructors static int gconstr_value = 0; -__attribute__((constructor)) +__attribute__((constructor, used)) static void self_test_gconstr() { gconstr_value = 1; } @@ -38,8 +38,6 @@ static uint32_t crc_ro = CRC32_BEGIN(); static uint32_t generate_ro_crc() noexcept { extern char _TEXT_START_; - extern char _TEXT_END_; - extern char _RODATA_START_; extern char _RODATA_END_; return crc32_fast(&_TEXT_START_, &_RODATA_END_ - &_TEXT_START_); } From 7c856f8ad08e4437e233b65ff3ec06e8ea0f420c Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 5 Sep 2018 14:25:28 +0200 Subject: [PATCH 585/723] kernel: Prevent ELF symbol checksums being overwritten by GC --- src/kernel/elf.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index 8e132459af..c52289046e 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -195,8 +195,10 @@ class ElfTables SymTab symtab; StrTab strtab; - uint32_t checksum_syms = 0; - uint32_t checksum_strs = 0; + /* NOTE: DON'T INITIALIZE */ + uint32_t checksum_syms; + uint32_t checksum_strs; + /* NOTE: DON'T INITIALIZE */ friend void elf_protect_symbol_areas(); }; static ElfTables parser; From 6dcec7f6e9ce79b18d7c57e95943441cb90c7fcf Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 5 Sep 2018 14:25:54 +0200 Subject: [PATCH 586/723] cmake: Remove -MMD, redundant and unused --- CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1b719bd40d..6721848798 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -140,12 +140,12 @@ enable_language(ASM_NASM) # initialize C and C++ compiler flags if (CMAKE_COMPILER_IS_GNUCC) # gcc/g++ settings - set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c") - set(CMAKE_C_FLAGS " -MMD ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c") + set(CMAKE_CXX_FLAGS "${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c") + set(CMAKE_C_FLAGS " ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c") else() # these kinda work with llvm - set(CMAKE_CXX_FLAGS "-MMD ${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c") - set(CMAKE_C_FLAGS "-MMD ${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c") + set(CMAKE_CXX_FLAGS "${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c") + set(CMAKE_C_FLAGS "${CAPABS} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c") endif() # either download or cross-compile needed libraries From 8adff51465e67a750ee7f8b98911496c34518b35 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 5 Sep 2018 16:04:55 +0200 Subject: [PATCH 587/723] x86: Disable RO sections sanity check --- src/platform/x86_pc/sanity_checks.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/x86_pc/sanity_checks.cpp b/src/platform/x86_pc/sanity_checks.cpp index cfc4f12d9f..498e3fdf5f 100644 --- a/src/platform/x86_pc/sanity_checks.cpp +++ b/src/platform/x86_pc/sanity_checks.cpp @@ -22,7 +22,7 @@ #include #include -#define ENABLE_CRC_RO +//#define ENABLE_CRC_RO // Global constructors static int gconstr_value = 0; From b02265e7987638dca5d0218a90624b92313a8861 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 6 Sep 2018 14:04:08 +0200 Subject: [PATCH 588/723] linux: Add build&run helper scripts --- CMakeLists.txt | 6 ++- etc/linux/lxp-callgraph | 9 +++++ etc/linux/lxp-debug | 4 ++ etc/linux/lxp-gprof | 3 ++ .../websockets/run.sh => etc/linux/lxp-run | 24 +++++++----- examples/demo_linux/run.sh | 9 ----- .../CMakeLists.txt | 0 .../{demo_linux => userspace_demo}/README.md | 0 examples/userspace_demo/run.sh | 3 ++ .../service.cpp | 0 test/linux/router/run.sh | 9 ----- test/linux/tcp/callgraph.sh | 12 ------ test/linux/tcp/run.sh | 37 ------------------- test/linux/websockets/callgraph.sh | 12 ------ test/linux/websockets/debug.sh | 5 --- 15 files changed, 39 insertions(+), 94 deletions(-) create mode 100755 etc/linux/lxp-callgraph create mode 100755 etc/linux/lxp-debug create mode 100755 etc/linux/lxp-gprof rename test/linux/websockets/run.sh => etc/linux/lxp-run (53%) delete mode 100755 examples/demo_linux/run.sh rename examples/{demo_linux => userspace_demo}/CMakeLists.txt (100%) rename examples/{demo_linux => userspace_demo}/README.md (100%) create mode 100755 examples/userspace_demo/run.sh rename examples/{demo_linux => userspace_demo}/service.cpp (100%) delete mode 100755 test/linux/router/run.sh delete mode 100755 test/linux/tcp/callgraph.sh delete mode 100755 test/linux/tcp/run.sh delete mode 100755 test/linux/websockets/callgraph.sh delete mode 100755 test/linux/websockets/debug.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index 6721848798..2c1c3953c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -280,8 +280,12 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/cmake/elf-toolchain.cmake DESTINATION # Install seed install(DIRECTORY seed/ DESTINATION includeos/seed) -# Install boot util +# Install executable scripts install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/boot DESTINATION bin) +install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-run DESTINATION bin) +install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-callgraph DESTINATION bin) +install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-debug DESTINATION bin) +install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-gprof DESTINATION bin) # Install scripts install(PROGRAMS diff --git a/etc/linux/lxp-callgraph b/etc/linux/lxp-callgraph new file mode 100755 index 0000000000..b401409e75 --- /dev/null +++ b/etc/linux/lxp-callgraph @@ -0,0 +1,9 @@ +#! /bin/bash +BINARY="`cat build/binary.txt`" +echo -e "\n>>> Installing dependencies" +pip -q install gprof2dot +sudo apt install -y gprof graphviz + +echo -e "\n>>> Generating callgraph.svg" +gprof build/$BINARY | gprof2dot | dot -Tsvg -o callgraph.svg +firefox callgraph.svg diff --git a/etc/linux/lxp-debug b/etc/linux/lxp-debug new file mode 100755 index 0000000000..04eb5f70ac --- /dev/null +++ b/etc/linux/lxp-debug @@ -0,0 +1,4 @@ +#! /bin/bash +echo -e "\n>>> Running service with debugging" +DBG="ON" +source lxp-run diff --git a/etc/linux/lxp-gprof b/etc/linux/lxp-gprof new file mode 100755 index 0000000000..b210868478 --- /dev/null +++ b/etc/linux/lxp-gprof @@ -0,0 +1,3 @@ +#! /bin/bash +echo -e "\n>>> Running service with profiling" +source lxp-run gprof diff --git a/test/linux/websockets/run.sh b/etc/linux/lxp-run similarity index 53% rename from test/linux/websockets/run.sh rename to etc/linux/lxp-run index 5fe9056068..36e390c44a 100755 --- a/test/linux/websockets/run.sh +++ b/etc/linux/lxp-run @@ -1,16 +1,13 @@ #!/bin/bash - set -e - -export GPROF="OFF" -export DBG="OFF" - -export num_jobs=${num_jobs:--j32} +export GPROF=${GPROF:-"OFF"} +export DBG=${DBG:-"OFF"} +export NUM_JOBS=${NUM_JOBS:--j8} function make_linux(){ pushd $INCLUDEOS_SRC/linux/build cmake -DGPROF=$GPROF -DDEBUGGING=$DBG .. - make $num_jobs install + make $NUM_JOBS install popd } @@ -18,7 +15,7 @@ function make_service(){ mkdir -p build pushd build cmake -DGPROF=$GPROF -DDEBUGGING=$DBG .. - make $num_jobs + make $NUM_JOBS popd } @@ -30,10 +27,19 @@ then GPROF="ON" fi +# check that at least there is a cmake script here +if [ ! -f CMakeLists.txt ]; then + echo "There must be at least a CMakeLists.txt in service folder" + exit 1 +fi make_linux make_service #sudo mknod /dev/net/tap c 10 200 BINARY=build/"`cat build/binary.txt`" -sudo $BINARY +if [ $DBG = "ON" ]; then + sudo gdb $BINARY +else + sudo $BINARY +fi diff --git a/examples/demo_linux/run.sh b/examples/demo_linux/run.sh deleted file mode 100755 index fed2702613..0000000000 --- a/examples/demo_linux/run.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -sudo mknod /dev/net/tap c 10 200 -set -e -mkdir -p build -pushd build -cmake .. -make -j4 -popd -sudo ./build/linux_demo diff --git a/examples/demo_linux/CMakeLists.txt b/examples/userspace_demo/CMakeLists.txt similarity index 100% rename from examples/demo_linux/CMakeLists.txt rename to examples/userspace_demo/CMakeLists.txt diff --git a/examples/demo_linux/README.md b/examples/userspace_demo/README.md similarity index 100% rename from examples/demo_linux/README.md rename to examples/userspace_demo/README.md diff --git a/examples/userspace_demo/run.sh b/examples/userspace_demo/run.sh new file mode 100755 index 0000000000..48d0ac4333 --- /dev/null +++ b/examples/userspace_demo/run.sh @@ -0,0 +1,3 @@ +#!/bin/bash +sudo mknod /dev/net/tap c 10 200 +lxp-run diff --git a/examples/demo_linux/service.cpp b/examples/userspace_demo/service.cpp similarity index 100% rename from examples/demo_linux/service.cpp rename to examples/userspace_demo/service.cpp diff --git a/test/linux/router/run.sh b/test/linux/router/run.sh deleted file mode 100755 index 98fa959a55..0000000000 --- a/test/linux/router/run.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -sudo mknod /dev/net/tap c 10 200 -set -e -mkdir -p build -pushd build -cmake .. -make -j4 -popd -sudo ./build/linux_router diff --git a/test/linux/tcp/callgraph.sh b/test/linux/tcp/callgraph.sh deleted file mode 100755 index 4ed8e1114a..0000000000 --- a/test/linux/tcp/callgraph.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/bash - - -echo -e "\n>>> Installing dependencies" -pip -q install gprof2dot -sudo apt install gprof graphviz - -echo -e "\n>>> Running tcp test service with profiling" -./run.sh gprof - -echo -e "\n>>> Generating graph png" -gprof ./build/linux_tcp | gprof2dot | dot -Tpng -o tcp_callgraph.png diff --git a/test/linux/tcp/run.sh b/test/linux/tcp/run.sh deleted file mode 100755 index 8156074996..0000000000 --- a/test/linux/tcp/run.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -set -e - -export GPROF="OFF" - -export num_jobs=${num_jobs:--j4} - -function make_linux(){ - pushd $INCLUDEOS_SRC/linux/build - cmake -DGPROF=$GPROF .. - make $num_jobs install - popd -} - -function make_service(){ - mkdir -p build - pushd build - cmake -DGPROF=$GPROF .. - make $num_jobs - popd -} - -if [ -z "$1" ] -then - GPROF="OFF" -elif [ "$1" == "gprof" ] -then - GPROF="ON" -fi - - -make_linux -make_service - -#sudo mknod /dev/net/tap c 10 200 -./build/linux_tcp diff --git a/test/linux/websockets/callgraph.sh b/test/linux/websockets/callgraph.sh deleted file mode 100755 index 4ed8e1114a..0000000000 --- a/test/linux/websockets/callgraph.sh +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/bash - - -echo -e "\n>>> Installing dependencies" -pip -q install gprof2dot -sudo apt install gprof graphviz - -echo -e "\n>>> Running tcp test service with profiling" -./run.sh gprof - -echo -e "\n>>> Generating graph png" -gprof ./build/linux_tcp | gprof2dot | dot -Tpng -o tcp_callgraph.png diff --git a/test/linux/websockets/debug.sh b/test/linux/websockets/debug.sh deleted file mode 100755 index 4c46625cb7..0000000000 --- a/test/linux/websockets/debug.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -pushd build -make -j8 -popd -sudo gdb build/websockets From eef58c828d26c4ef5d0df07de1a61355615052e7 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 6 Sep 2018 15:45:26 +0200 Subject: [PATCH 589/723] linux: Update the Linux userspace examples --- examples/TCP_perf/linux/README.md | 7 +++++++ examples/TCP_perf/linux/run.sh | 10 ---------- examples/userspace_demo/README.md | 10 +++++----- examples/userspace_demo/run.sh | 3 --- 4 files changed, 12 insertions(+), 18 deletions(-) create mode 100644 examples/TCP_perf/linux/README.md delete mode 100755 examples/TCP_perf/linux/run.sh delete mode 100755 examples/userspace_demo/run.sh diff --git a/examples/TCP_perf/linux/README.md b/examples/TCP_perf/linux/README.md new file mode 100644 index 0000000000..93099ee600 --- /dev/null +++ b/examples/TCP_perf/linux/README.md @@ -0,0 +1,7 @@ +## How to run + +1. Install the OS + +2. Create TAP: `sudo mknod /dev/net/tap c 10 200` + +3. `lxp-run` diff --git a/examples/TCP_perf/linux/run.sh b/examples/TCP_perf/linux/run.sh deleted file mode 100755 index 74a29c3252..0000000000 --- a/examples/TCP_perf/linux/run.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e -mkdir -p build -pushd build -cmake .. -make -j4 -popd -set +e -sudo mknod /dev/net/tap c 10 200 -sudo ./build/tcp_perf diff --git a/examples/userspace_demo/README.md b/examples/userspace_demo/README.md index eee01ef99d..d2a1284121 100644 --- a/examples/userspace_demo/README.md +++ b/examples/userspace_demo/README.md @@ -5,12 +5,12 @@ Build and install the linux userspace library first, from the linux folder: mkdir -p build && pushd build && cmake .. -DCMAKE_INSTALL_PREFIX=$INCLUDEOS_PREFIX && make -j4 install && popd ``` -Build with cmake & make, then run: +Then run: ``` -mkdir -p build && pushd build && cmake .. && make && popd -./build/demo_example +sudo mknod /dev/net/tap c 10 200 +lxp-run ``` -This demo-service should start an instance of IncludeOS that brings up a minimal web service on port 80 with static content. +This demo service should start an instance of IncludeOS that brings up a minimal web service on port 80 with static content. -The default static IP is 10.0.0.2. +The default static IP is 10.0.0.42. diff --git a/examples/userspace_demo/run.sh b/examples/userspace_demo/run.sh deleted file mode 100755 index 48d0ac4333..0000000000 --- a/examples/userspace_demo/run.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -sudo mknod /dev/net/tap c 10 200 -lxp-run From 3def5e57b8b8aca7c1cb9c6b91d02de58d1a3f39 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 6 Sep 2018 16:28:41 +0200 Subject: [PATCH 590/723] linux: Fail on first failure when making callgraph --- etc/linux/lxp-callgraph | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etc/linux/lxp-callgraph b/etc/linux/lxp-callgraph index b401409e75..7aeb543103 100755 --- a/etc/linux/lxp-callgraph +++ b/etc/linux/lxp-callgraph @@ -1,5 +1,7 @@ #! /bin/bash BINARY="`cat build/binary.txt`" +set-e + echo -e "\n>>> Installing dependencies" pip -q install gprof2dot sudo apt install -y gprof graphviz From 50daaffc971ff2da8e62aaf3a8810cfd59544801 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 6 Sep 2018 16:30:43 +0200 Subject: [PATCH 591/723] tcp: Use unordered_map for Connections, add hashes for Socket --- api/net/socket.hpp | 27 ++++++++++++++++++++------- api/net/tcp/tcp.hpp | 2 +- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/api/net/socket.hpp b/api/net/socket.hpp index 65a36ca256..d137219bf6 100644 --- a/api/net/socket.hpp +++ b/api/net/socket.hpp @@ -207,13 +207,26 @@ struct Quadruple { } //< namespace net namespace std { - template<> - struct hash { - public: - size_t operator () (const net::Socket& key) const noexcept { - const auto h1 = std::hash{}(key.address().v6()); - const auto h2 = std::hash{}(key.port()); - return h1 ^ h2; + template<> struct hash + { + typedef net::Socket argument_type; + typedef std::size_t result_type; + result_type operator()(argument_type const& s) const noexcept + { + return s.address().v6().i64[0] + ^ s.address().v6().i64[1] + ^ s.port(); + } + }; + + template<> struct hash> + { + typedef std::pair argument_type; + typedef std::size_t result_type; + result_type operator()(argument_type const& s) const noexcept + { + return std::hash{}(s.first) + ^ std::hash{}(s.second); } }; diff --git a/api/net/tcp/tcp.hpp b/api/net/tcp/tcp.hpp index 17a8626d48..826753b85f 100644 --- a/api/net/tcp/tcp.hpp +++ b/api/net/tcp/tcp.hpp @@ -58,7 +58,7 @@ namespace net { private: using Listeners = std::map>; - using Connections = std::map; + using Connections = std::unordered_map; public: /////// TCP Stuff - Relevant to the protocol ///// From 0d99452aa7b013c260d598d9aff424e94167a94a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 6 Sep 2018 16:31:06 +0200 Subject: [PATCH 592/723] ip4: Temporarily remove Expects on ip_data_length() --- api/net/ip4/packet_ip4.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/net/ip4/packet_ip4.hpp b/api/net/ip4/packet_ip4.hpp index e804ccf9ec..4352285f2a 100644 --- a/api/net/ip4/packet_ip4.hpp +++ b/api/net/ip4/packet_ip4.hpp @@ -99,7 +99,7 @@ namespace net { /** Get IP data length. */ uint16_t ip_data_length() const noexcept { - Expects(size() and static_cast(size()) >= sizeof(ip4::Header)); + //Expects(size() and static_cast(size()) >= sizeof(ip4::Header)); return size() - ip_header_length(); } From 21f0bf3004d90fe3b077ee934c1acfdee563e633 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 7 Sep 2018 10:06:44 +0200 Subject: [PATCH 593/723] git: Ignore callgraph.svg --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 96eadfbcc6..1f6e96d22b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ .DS_Store nbproject dummy.disk +callgraph.svg !Dockerfile From c82c94119390862025ae14bdb8148c9db897969b Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 7 Sep 2018 10:43:45 +0200 Subject: [PATCH 594/723] linux: Add proper plugin support --- .gitignore | 1 + cmake/linux.service.cmake | 5 ++-- linux/CMakeLists.txt | 1 + linux/src/plugins/CMakeLists.txt | 43 ++++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 linux/src/plugins/CMakeLists.txt diff --git a/.gitignore b/.gitignore index 1f6e96d22b..a050b2c304 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ nbproject dummy.disk callgraph.svg +massif.out.* !Dockerfile diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index fe82840f1b..341400216f 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -3,7 +3,7 @@ #################################### set(CMAKE_CXX_STANDARD 17) -set(COMMON "-O2 -march=native -Wall -Wextra") +set(COMMON "-g -O2 -march=native -Wall -Wextra") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") @@ -42,7 +42,6 @@ add_definitions("-DARCH=${ARCH}" "-DARCH_${ARCH}") add_definitions(-DOS_TERMINATE_ON_CONTRACT_VIOLATION) add_definitions(-DARP_PASSTHROUGH) add_definitions(-DNO_DEBUG) -add_definitions(-DINCLUDEOS_SINGLE_THREADED) add_definitions(-DSERVICE=\"\\\"${BINARY}\\\"\") add_definitions(-DSERVICE_NAME=\"\\\"${SERVICE_NAME}\\\"\") add_definitions(-DUSERSPACE_LINUX) @@ -62,7 +61,7 @@ add_executable(service ${SOURCES} ${IOSPATH}/src/service_name.cpp) set_target_properties(service PROPERTIES OUTPUT_NAME ${BINARY}) set(LPATH ${IOSPATH}/linux) -set(PLUGIN_LOC "${IOSPATH}/${ARCH}/plugins") +set(PLUGIN_LOC "${IOSPATH}/linux/plugins") set(DRIVER_LOC "${IOSPATH}/${ARCH}/drivers") # IncludeOS plugins diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index 231a2130f1..56b4eac1ac 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -51,4 +51,5 @@ include_directories(../mod) include_directories(../mod/GSL) add_subdirectory(src) +add_subdirectory(src/plugins) add_subdirectory(userspace) diff --git a/linux/src/plugins/CMakeLists.txt b/linux/src/plugins/CMakeLists.txt new file mode 100644 index 0000000000..cc3fa95c3f --- /dev/null +++ b/linux/src/plugins/CMakeLists.txt @@ -0,0 +1,43 @@ +# +# Build and install plugins as libraries +# +set (IOS ${CMAKE_SOURCE_DIR}/..) +set (PDIR "${IOS}/src/plugins") + +# make LiveUpdate visible to plugins +include_directories(${IOS}/lib/LiveUpdate) +include_directories(${IOS}/mod/rapidjson/include) + +#add_library(system_log STATIC "${PDIR}/system_log.cpp") +add_library(syslogd STATIC "${PDIR}/syslogd.cpp") +add_library(unik STATIC "${PDIR}/unik.cpp") +#add_library(example STATIC "${PDIR}/example.cpp") +add_library(autoconf STATIC "${PDIR}/autoconf.cpp") +add_library(terminal STATIC "${PDIR}/terminal.cpp") +add_library(terminal_liu STATIC "${PDIR}/terminal.cpp") +set_target_properties(terminal_liu PROPERTIES COMPILE_FLAGS "-DUSE_LIVEUPDATE") +add_library(nacl STATIC "${PDIR}/nacl.cpp") +add_library(vfs STATIC "${PDIR}/vfs.cpp") +add_library(field_medic STATIC + "${PDIR}/field_medic/fieldmedic.cpp" + "${PDIR}/field_medic/diag.cpp") +add_library(syslog STATIC "${PDIR}/syslog.cpp") + +# +# Installation +# +set(CMAKE_INSTALL_MESSAGE LAZY) # to avoid spam + +install(TARGETS + #system_log + syslogd + unik + #example + autoconf + terminal + terminal_liu + nacl + vfs + field_medic + syslog + DESTINATION includeos/linux/plugins) From 1a10145045dd744cf2a1fbcc6c002c6c15be3f27 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 7 Sep 2018 11:32:13 +0200 Subject: [PATCH 595/723] examples: Replace some autos to avoid GCC bug --- examples/TCP_perf/service.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/TCP_perf/service.cpp b/examples/TCP_perf/service.cpp index 7ff2dba31f..e9fe6657c1 100644 --- a/examples/TCP_perf/service.cpp +++ b/examples/TCP_perf/service.cpp @@ -101,6 +101,7 @@ void stop_measure() double mbits = (received/(1024*1024)*8) / durs; printf("Duration: %.2fs - Payload: %lu/%u MB - %.2f MBit/s\n", durs, received/(1024*1024), SIZE/(1024*1024), mbits); + OS::shutdown(); } void Service::start() {} @@ -148,10 +149,9 @@ void Service::ready() conn->close(); }); - tcp.listen(1338).on_connect([](auto conn) + tcp.listen(1338).on_connect([](net::tcp::Connection_ptr conn) { using namespace std::chrono; - printf("%s connected. Receiving file %u MB\n", conn->remote().to_string().c_str(), SIZE/(1024*1024)); start_measure(); @@ -160,7 +160,8 @@ void Service::ready() { }); - conn->on_disconnect([] (auto self, auto reason) + conn->on_disconnect([] (net::tcp::Connection_ptr self, + net::tcp::Connection::Disconnect reason) { (void) reason; if(const auto bytes_sacked = self->bytes_sacked(); bytes_sacked) From 4aed78caf54384630ab7e4048195b5872ae167d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 7 Sep 2018 14:18:26 +0200 Subject: [PATCH 596/723] tcp: Only process (recv) data if there is a read_request (created by on_read) --- src/net/tcp/connection.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index c27786d21b..dc0d51857d 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -694,7 +694,7 @@ void Connection::recv_data(const Packet_view& in) // If we had packet loss before (and SACK is on) // we need to clear up among the blocks // and increase the total amount of bytes acked - if(UNLIKELY(sack_list)) + if(UNLIKELY(sack_list != nullptr)) { const auto res = sack_list->new_valid_ack(in.seq(), length); // if any bytes are cleared up in sack, increase expected sequence number @@ -708,18 +708,25 @@ void Connection::recv_data(const Packet_view& in) length = res.length; } + // make sure to mark the data as recveied (ACK) before putting in buffer, // since user callback can result in sending new data, which means we // want to ACK the data recv at the same time cb.RCV.NXT += length; - const auto recv = read_request->insert(in.seq(), in.tcp_data(), length, in.isset(PSH)); - // this ensures that the data we ACK is actually put in our buffer. - Ensures(recv == length); + // only actually recv the data if there is a read request (created with on_read) + if(read_request != nullptr) + { + const auto recv = read_request->insert(in.seq(), in.tcp_data(), length, in.isset(PSH)); + // this ensures that the data we ACK is actually put in our buffer. + Ensures(recv == length); + } } // Packet out of order else if((in.seq() - cb.RCV.NXT) < cb.RCV.WND) { - recv_out_of_order(in); + // only accept the data if we have a read request + if(read_request != nullptr) + recv_out_of_order(in); } // User callback didnt result in transmitting an ACK From 2813509133d71ba89aceda3d250b862f2b645752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 7 Sep 2018 14:45:57 +0200 Subject: [PATCH 597/723] tcp: Don't send data before handshake is fulfilled with lonely ACK --- src/net/tcp/connection_states.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/net/tcp/connection_states.cpp b/src/net/tcp/connection_states.cpp index be854997e5..7d477b785b 100644 --- a/src/net/tcp/connection_states.cpp +++ b/src/net/tcp/connection_states.cpp @@ -851,14 +851,13 @@ State::Result Connection::SynSent::handle(Connection& tcp, Packet_view& in) { tcp.rttm.RTO = RTTM::seconds(3.0); } + // make sure to send an ACK to fullfil the handshake + // before calling user callback in case of user write + tcp.send_ack(); + tcp.set_state(Connection::Established::instance()); - const seq_t snd_nxt = tcb.SND.NXT; - tcp.signal_connect(); // NOTE: User callback - if(tcb.SND.NXT == snd_nxt) - { - tcp.send_ack(); - } + tcp.signal_connect(); // NOTE: User callback if(tcp.has_doable_job()) tcp.writeq_push(); From 3557ce79371be6eddb767263fd289abcc3589442 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 11:02:11 +0200 Subject: [PATCH 598/723] install: Add Linux Platform option --- install.sh | 18 ++++++++++++++++++ test/testrunner.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index c4af520f68..0775956590 100755 --- a/install.sh +++ b/install.sh @@ -10,6 +10,8 @@ export INCLUDEOS_SRC=${INCLUDEOS_SRC:-`pwd`} export INCLUDEOS_PREFIX=${INCLUDEOS_PREFIX:-/usr/local} # Enable compilation of tests in cmake (default: OFF) export INCLUDEOS_ENABLE_TEST=${INCLUDEOS_ENABLE_TEST:-OFF} +# Enable building of the Linux platform (default: OFF) +export INCLUDEOS_ENABLE_LXP=${INCLUDEOS_ENABLE_LXP:-OFF} # Set CPU-architecture (default x86_64) export ARCH=${ARCH:-x86_64} # Enable threading @@ -175,6 +177,7 @@ printf " %-25s %-25s %s\n"\ "INCLUDEOS_PREFIX" "Install location" "$INCLUDEOS_PREFIX"\ "ARCH" "CPU Architecture" "$ARCH"\ "INCLUDEOS_ENABLE_TEST" "Enable test compilation" "$INCLUDEOS_ENABLE_TEST"\ + "INCLUDEOS_ENABLE_LXP" "Linux Userspace platform" "$INCLUDEOS_ENABLE_LXP"\ "INCLUDEOS_THREADING" "Enable threading / SMP" "$INCLUDEOS_THREADING" # Give user option to evaluate install options @@ -250,6 +253,21 @@ if ! ./etc/build_chainloader.sh; then fi +# Install Linux platform +if [ "$INCLUDEOS_ENABLE_LXP" = "ON" ]; then + printf "\n\n>>> Installing Linux Userspace platform\n" + pushd linux + mkdir -p build + pushd build + cmake .. + make -j$num_jobs + make install + popd + popd +else + printf "\n\n>>> Not installing Linux Userspace platform\n" +fi + ############################################################ # INSTALL FINISHED: ############################################################ diff --git a/test/testrunner.py b/test/testrunner.py index 5f308ea7c7..7e086912d0 100755 --- a/test/testrunner.py +++ b/test/testrunner.py @@ -20,7 +20,7 @@ startdir = os.getcwd() -test_categories = ['fs', 'hw', 'kernel', 'mod', 'net', 'performance', 'plugin', 'posix', 'stl', 'util'] +test_categories = ['fs', 'hw', 'kernel', 'linux', 'mod', 'net', 'performance', 'plugin', 'posix', 'stl', 'util'] test_types = ['integration', 'stress', 'unit', 'misc'] """ From 22ffec65cf0e8442b0706d0f057426ae9ed95a26 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 11:57:12 +0200 Subject: [PATCH 599/723] net: Add reason on static_assert() --- api/net/ip4/addr.hpp | 2 +- api/net/ip6/addr.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/net/ip4/addr.hpp b/api/net/ip4/addr.hpp index 3c92e120ec..07b13a4481 100644 --- a/api/net/ip4/addr.hpp +++ b/api/net/ip4/addr.hpp @@ -387,7 +387,7 @@ struct Addr { uint32_t whole; } __attribute__((packed)); //< struct Addr -static_assert(sizeof(Addr) == 4); +static_assert(sizeof(Addr) == 4, "Must be 4 bytes in size"); } //< namespace ip4 diff --git a/api/net/ip6/addr.hpp b/api/net/ip6/addr.hpp index b5d336170c..3a6218276e 100644 --- a/api/net/ip6/addr.hpp +++ b/api/net/ip6/addr.hpp @@ -279,7 +279,7 @@ struct Addr { std::array i8; }; } __attribute__((packed)); //< struct Addr -static_assert(sizeof(Addr) == 16); +static_assert(sizeof(Addr) == 16, "Must be 16 bytes in size"); } //< namespace ip6 } //< namespace net From 4c17e76f5eaf713be4d16883d3fb39ba9c09853a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 11:57:29 +0200 Subject: [PATCH 600/723] install: Use GCC 7 with Linux Platform --- install.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index 0775956590..95a8fbdfd9 100755 --- a/install.sh +++ b/install.sh @@ -259,9 +259,8 @@ if [ "$INCLUDEOS_ENABLE_LXP" = "ON" ]; then pushd linux mkdir -p build pushd build - cmake .. - make -j$num_jobs - make install + CXX=g++-7 CC=gcc-7 cmake .. + make ${num_jobs:="-j 4"} install popd popd else From fc4281a45f86e62764a7f2e6137a2f46fb766f86 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 12:29:38 +0200 Subject: [PATCH 601/723] linux: Work-around for old CMake --- linux/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index 56b4eac1ac..be69503702 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required (VERSION 3.0.2) +cmake_minimum_required (VERSION 3.5.1) project (userland_tests C CXX) # create OS version string from git describe (used in CXX flags) @@ -7,8 +7,8 @@ execute_process(COMMAND git describe --dirty OUTPUT_VARIABLE OS_VERSION) string(STRIP ${OS_VERSION} OS_VERSION) -set(CMAKE_CXX_STANDARD 17) -set(COMMON "-g -O2 -march=native -Wall -Wextra") +#set(CMAKE_CXX_STANDARD 17) +set(COMMON "-g -O2 -march=native -Wall -Wextra -std=c++17") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") From 70198a10f640dd646e6debb599b94c142b209482 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 14:08:52 +0200 Subject: [PATCH 602/723] install: Use install prefix properly --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 95a8fbdfd9..749f6c9c63 100755 --- a/install.sh +++ b/install.sh @@ -259,7 +259,7 @@ if [ "$INCLUDEOS_ENABLE_LXP" = "ON" ]; then pushd linux mkdir -p build pushd build - CXX=g++-7 CC=gcc-7 cmake .. + CXX=g++-7 CC=gcc-7 cmake .. -DCMAKE_INSTALL_PREFIX=$INCLUDEOS_PREFIX make ${num_jobs:="-j 4"} install popd popd From 17e80b944351d9c839ff0c54c81aa5c098535f61 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 14:09:10 +0200 Subject: [PATCH 603/723] linux: Add TCP test for testrunner --- test/linux/tcp/service.cpp | 2 +- test/linux/tcp/test.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100755 test/linux/tcp/test.py diff --git a/test/linux/tcp/service.cpp b/test/linux/tcp/service.cpp index 9e336e2ec9..84c743a5dc 100644 --- a/test/linux/tcp/service.cpp +++ b/test/linux/tcp/service.cpp @@ -76,7 +76,7 @@ void Service::start() double time_sec = timediff.count()/1000.0; double mbps = ((count_bytes * 8) / (1024.0 * 1024.0)) / time_sec; - printf("Server reveived %zu Mb in %f sec. - %f Mbps \n", + printf("Server received %zu Mb in %f sec. - %f Mbps \n", count_bytes / (1024 * 1024), time_sec, mbps); for (const auto& stat : Statman::get()) diff --git a/test/linux/tcp/test.py b/test/linux/tcp/test.py new file mode 100755 index 0000000000..a306510f34 --- /dev/null +++ b/test/linux/tcp/test.py @@ -0,0 +1,11 @@ +#!/usr/bin/python +import subprocess +import os + +args = ['sudo', '-E', os.environ['INCLUDEOS_PREFIX'] + '/bin/lxp-run'] +res = subprocess.check_output(args) +text = res.decode('utf-8') +#print text +assert "TCP demo started" in text +assert "Server received" in text +print ">>> Linux Userspace TCP test success!" From bfc6f2e1c4c2c660155aea5f530b953e2f830609 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 14:52:05 +0200 Subject: [PATCH 604/723] linux: Fix compile and install prefix issues --- cmake/linux.service.cmake | 2 +- etc/linux/lxp-run | 2 +- linux/CMakeLists.txt | 6 +++--- test/linux/tcp/{test.py => test.sh} | 0 4 files changed, 5 insertions(+), 5 deletions(-) rename test/linux/tcp/{test.py => test.sh} (100%) diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index 341400216f..3825240759 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -12,7 +12,7 @@ option(GPROF "Enable profiling with gprof" OFF) option(SANITIZE "Enable undefined- and address sanitizers" OFF) option(ENABLE_LTO "Enable thinLTO for use with LLD" OFF) option(CUSTOM_BOTAN "Enable building with a local Botan" OFF) -option(STATIC_BUILD "Build a portable static executable" OFF) +option(STATIC_BUILD "Build a portable static executable" ON) option(STRIP_BINARY "Strip final binary to reduce size" OFF) option(USE_LLD "Allow linking against LTO archives" ON) diff --git a/etc/linux/lxp-run b/etc/linux/lxp-run index 36e390c44a..ee1e80f05c 100755 --- a/etc/linux/lxp-run +++ b/etc/linux/lxp-run @@ -6,7 +6,7 @@ export NUM_JOBS=${NUM_JOBS:--j8} function make_linux(){ pushd $INCLUDEOS_SRC/linux/build - cmake -DGPROF=$GPROF -DDEBUGGING=$DBG .. + cmake -DGPROF=$GPROF -DDEBUGGING=$DBG -DCMAKE_INSTALL_PREFIX=$INCLUDEOS_PREFIX .. make $NUM_JOBS install popd } diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index be69503702..389172a2f6 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -8,9 +8,9 @@ execute_process(COMMAND git describe --dirty string(STRIP ${OS_VERSION} OS_VERSION) #set(CMAKE_CXX_STANDARD 17) -set(COMMON "-g -O2 -march=native -Wall -Wextra -std=c++17") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") -set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") +set(COMMON "-g -O2 -march=native -Wall -Wextra") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${COMMON}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON}") option(DEBUGGING "Enable debugging" OFF) option(GPROF "Enable profiling with gprof" OFF) diff --git a/test/linux/tcp/test.py b/test/linux/tcp/test.sh similarity index 100% rename from test/linux/tcp/test.py rename to test/linux/tcp/test.sh From 0f0938c14604a190994f619c8cfbd6aeb3abe0ea Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 15:01:12 +0200 Subject: [PATCH 605/723] test: Update testrunner to run Linux tests --- test/linux/tcp/test.sh | 18 ++++++++---------- test/testrunner.py | 40 ++++++++++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/test/linux/tcp/test.sh b/test/linux/tcp/test.sh index a306510f34..4c3589db27 100755 --- a/test/linux/tcp/test.sh +++ b/test/linux/tcp/test.sh @@ -1,11 +1,9 @@ -#!/usr/bin/python -import subprocess -import os +#!/bin/bash +set -e +$INCLUDEOS_PREFIX/bin/lxp-run | grep 'Server received' -args = ['sudo', '-E', os.environ['INCLUDEOS_PREFIX'] + '/bin/lxp-run'] -res = subprocess.check_output(args) -text = res.decode('utf-8') -#print text -assert "TCP demo started" in text -assert "Server received" in text -print ">>> Linux Userspace TCP test success!" +if [ $? == 0 ]; then + echo ">>> Linux Userspace TCP test success!" +else + exit 1 +fi diff --git a/test/testrunner.py b/test/testrunner.py index 7e086912d0..70542d26d9 100755 --- a/test/testrunner.py +++ b/test/testrunner.py @@ -20,8 +20,8 @@ startdir = os.getcwd() -test_categories = ['fs', 'hw', 'kernel', 'linux', 'mod', 'net', 'performance', 'plugin', 'posix', 'stl', 'util'] -test_types = ['integration', 'stress', 'unit', 'misc'] +test_categories = ['fs', 'hw', 'kernel', 'mod', 'net', 'performance', 'plugin', 'posix', 'stl', 'util'] +test_types = ['integration', 'stress', 'unit', 'misc', 'linux'] """ Script used for running all the valid tests in the terminal. @@ -86,6 +86,10 @@ def __init__(self, path, clean=False, command=['python', '-u', 'test.py'], name= self.category_ = 'misc' self.type_ = 'misc' self.command_ = ['./test.sh'] + elif self.path_.split("/")[1] == 'linux': + self.category_ = 'linux' + self.type_ = 'linux' + self.command_ = ['./test.sh'] elif self.path_ == 'mod/gsl': self.category_ = 'mod' self.type_ = 'mod' @@ -211,15 +215,32 @@ def check_valid(self): self.skip_reason_ = None return + # Linux tests only need a test.sh + if self.type_ == "linux": + for f in ["CMakeLists.txt", "test.sh"]: + if not os.path.isfile(self.path_ + "/" + f): + self.skip_ = True + self.skip_reason_ = 'Missing required file: ' + f + return + self.skip_ = False + self.skip_reason_ = None + return + + # Figure out if the test should be skipped # Test 1 + if self.path_ in args.skip or self.category_ in args.skip: + self.skip_ = True + self.skip_reason_ = 'Defined by cmd line argument' + return + + # Test 2 valid, err = validate_tests.validate_test(self.path_, verb = False) if not valid: self.skip_ = True self.skip_reason_ = err return - # Test 2 - # Figure out if the test should be skipped + # Test 3 skip_json = json.loads(open("skipped_tests.json").read()) for skip in skip_json: if skip['name'] in self.path_: @@ -227,12 +248,6 @@ def check_valid(self): self.skip_reason_ = skip['reason'] return - # Test 3 - if self.path_ in args.skip or self.category_ in args.skip: - self.skip_ = True - self.skip_reason_ = 'Defined by cmd line argument' - return - self.skip_ = False self.skip_reason_ = None return @@ -371,7 +386,7 @@ def find_test_folders(): # Only look in folders listed as a test category if directory in test_types: - if directory == 'misc': + if directory == 'misc' or directory == 'linux': # For each subfolder in misc, register test for subdir in os.listdir("/".join(path)): path.append(subdir) @@ -522,9 +537,10 @@ def main(): integration_result = integration_tests([x for x in filtered_tests if x.type_ == "integration"]) stress_result = stress_test([x for x in filtered_tests if x.type_ == "stress"]) misc_result = misc_working([x for x in filtered_tests if x.type_ == "misc"]) + linux_result = misc_working([x for x in filtered_tests if x.type_ == "linux"]) # Print status from test run - status = max(integration_result, stress_result, misc_result) + status = max(integration_result, stress_result, misc_result, linux_result) if (status == 0): print pretty.SUCCESS(str(test_count - status) + " / " + str(test_count) + " tests passed, exiting with code 0") From 199df5ac9ab93334040c14f8abe897d87fa32e78 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 15:46:11 +0200 Subject: [PATCH 606/723] linux: Work-around for old CMake version --- cmake/linux.service.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index 3825240759..4fd89581ad 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -2,10 +2,10 @@ # Linux Userspace CMake script # #################################### -set(CMAKE_CXX_STANDARD 17) +#set(CMAKE_CXX_STANDARD 17) set(COMMON "-g -O2 -march=native -Wall -Wextra") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") -set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} ${COMMON}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${COMMON}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON}") option(DEBUGGING "Enable debugging" OFF) option(GPROF "Enable profiling with gprof" OFF) From a9fbb9e9f5d5ff7ecc5927020ad68bb3cce08565 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 16:07:27 +0200 Subject: [PATCH 607/723] test: Name linux platform section --- test/testrunner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/testrunner.py b/test/testrunner.py index 70542d26d9..bd45c7f917 100755 --- a/test/testrunner.py +++ b/test/testrunner.py @@ -275,7 +275,7 @@ def stress_test(stress_tests): return 1 if test.wait_status() else 0 -def misc_working(misc_tests): +def misc_working(misc_tests, test_type): global test_count test_count += len(misc_tests) if len(misc_tests) == 0: @@ -285,7 +285,7 @@ def misc_working(misc_tests): print pretty.WARNING("Misc test skipped") return 0 - print pretty.HEADER("Building " + str(len(misc_tests)) + " misc") + print pretty.HEADER("Building " + str(len(misc_tests)) + " " + str(test_type)) fail_count = 0 for test in misc_tests: @@ -536,8 +536,8 @@ def main(): # Run the tests integration_result = integration_tests([x for x in filtered_tests if x.type_ == "integration"]) stress_result = stress_test([x for x in filtered_tests if x.type_ == "stress"]) - misc_result = misc_working([x for x in filtered_tests if x.type_ == "misc"]) - linux_result = misc_working([x for x in filtered_tests if x.type_ == "linux"]) + misc_result = misc_working([x for x in filtered_tests if x.type_ == "misc"], "misc") + linux_result = misc_working([x for x in filtered_tests if x.type_ == "linux"], "linux platform") # Print status from test run status = max(integration_result, stress_result, misc_result, linux_result) From 2b1e0f8771ba7a883d943df75619b9d2d61fd485 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 16:07:40 +0200 Subject: [PATCH 608/723] linux: Disable LLD by default --- cmake/linux.service.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index 4fd89581ad..c33d28b24b 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -14,7 +14,7 @@ option(ENABLE_LTO "Enable thinLTO for use with LLD" OFF) option(CUSTOM_BOTAN "Enable building with a local Botan" OFF) option(STATIC_BUILD "Build a portable static executable" ON) option(STRIP_BINARY "Strip final binary to reduce size" OFF) -option(USE_LLD "Allow linking against LTO archives" ON) +option(USE_LLD "Allow linking against LTO archives" OFF) if(DEBUGGING) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") From f1ac2bce6952f2fe6ad2481eaabb8a4a96532945 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 17:59:29 +0200 Subject: [PATCH 609/723] linux: Use GCC explicitly when building test --- test/linux/tcp/test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/linux/tcp/test.sh b/test/linux/tcp/test.sh index 4c3589db27..be328bf5b6 100755 --- a/test/linux/tcp/test.sh +++ b/test/linux/tcp/test.sh @@ -1,5 +1,7 @@ #!/bin/bash set -e +export CC=gcc-7 +export CXX=g++-7 $INCLUDEOS_PREFIX/bin/lxp-run | grep 'Server received' if [ $? == 0 ]; then From f05366a9235137b5e63fb10dca33c923a5bd6b6a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 10 Sep 2018 18:20:02 +0200 Subject: [PATCH 610/723] linux: Add LTO support for GCC --- cmake/linux.service.cmake | 11 ++++++++--- linux/CMakeLists.txt | 21 +++++++++++++++++---- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index c33d28b24b..6525927f03 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -10,7 +10,7 @@ set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON}") option(DEBUGGING "Enable debugging" OFF) option(GPROF "Enable profiling with gprof" OFF) option(SANITIZE "Enable undefined- and address sanitizers" OFF) -option(ENABLE_LTO "Enable thinLTO for use with LLD" OFF) +option(ENABLE_LTO "Enable LTO for use with Clang/GCC" OFF) option(CUSTOM_BOTAN "Enable building with a local Botan" OFF) option(STATIC_BUILD "Build a portable static executable" ON) option(STRIP_BINARY "Strip final binary to reduce size" OFF) @@ -21,8 +21,13 @@ if(DEBUGGING) endif() if (ENABLE_LTO) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") - set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto") + set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} -flto") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") + set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") + endif() endif() if(GPROF) diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index 389172a2f6..2e76c45fef 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -12,19 +12,32 @@ set(COMMON "-g -O2 -march=native -Wall -Wextra") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${COMMON}") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON}") +option(PERFORMANCE "Enable maximum performance" OFF) option(DEBUGGING "Enable debugging" OFF) option(GPROF "Enable profiling with gprof" OFF) option(SANITIZE "Enable undefined- and address sanitizers" OFF) -option(ENABLE_LTO "Enable thinLTO for use with LLD" OFF) +option(ENABLE_LTO "Enable LTO for use with Clang/GCC" OFF) option(CUSTOM_BOTAN "Enable building with a local Botan" OFF) -if(DEBUGGING) +if (PERFORMANCE) + if (DEBUGGING) + message(FATAL_ERROR "You can not mix PERFORMANCE and DEBUGGING") + endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Ofast") +endif() + +if (DEBUGGING) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") endif() if (ENABLE_LTO) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") - set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto") + set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} -flto") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") + set(CMAKE_C_FLAGS "${CMAKE_CXX_FLAGS} -flto=thin") + endif() endif() if(GPROF) From 4fbb908046ad7db0e0fff43e6ce53dec7cd15225 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 11 Sep 2018 11:12:02 +0200 Subject: [PATCH 611/723] install: Exit on failure to build Linux platform --- install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install.sh b/install.sh index 749f6c9c63..90f692f0c4 100755 --- a/install.sh +++ b/install.sh @@ -259,8 +259,10 @@ if [ "$INCLUDEOS_ENABLE_LXP" = "ON" ]; then pushd linux mkdir -p build pushd build + set -e CXX=g++-7 CC=gcc-7 cmake .. -DCMAKE_INSTALL_PREFIX=$INCLUDEOS_PREFIX make ${num_jobs:="-j 4"} install + set +e popd popd else From 1a9ac1606faa2b0ee7133883ccc195965ecb6e04 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 11 Sep 2018 13:01:40 +0200 Subject: [PATCH 612/723] linux: Compile with frame-pointer intact when profiling --- cmake/linux.service.cmake | 2 +- linux/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index 6525927f03..6705ce81f4 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -31,7 +31,7 @@ if (ENABLE_LTO) endif() if(GPROF) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg -fno-omit-frame-pointer") endif() if(SANITIZE) diff --git a/linux/CMakeLists.txt b/linux/CMakeLists.txt index 2e76c45fef..fb14c90dab 100644 --- a/linux/CMakeLists.txt +++ b/linux/CMakeLists.txt @@ -41,7 +41,7 @@ if (ENABLE_LTO) endif() if(GPROF) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg -fno-omit-frame-pointer") endif() if(SANITIZE) From 5e06556115cc87c23d120b3c6c03ac7fd91485b2 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 11 Sep 2018 14:36:49 +0200 Subject: [PATCH 613/723] linux: Fix inconsistency with linker symbols --- linux/src/arch.cpp | 4 ++-- src/kernel/os.cpp | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/linux/src/arch.cpp b/linux/src/arch.cpp index ff66afdce1..987e75a968 100644 --- a/linux/src/arch.cpp +++ b/linux/src/arch.cpp @@ -12,8 +12,8 @@ weak_alias(__plugin_ctors_start, __plugin_ctors_end); ctor_t __service_ctors_start = default_ctor; weak_alias(__service_ctors_start, __service_ctors_end); -void* _ELF_START_; -void* _ELF_END_; +char _ELF_START_; +char _ELF_END_; uintptr_t OS::heap_max() noexcept { diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index 05df07d7b1..d355e2583f 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -41,12 +41,12 @@ using namespace util; extern "C" void* get_cpu_esp(); -extern uintptr_t _start; -extern uintptr_t _end; -extern uintptr_t _ELF_START_; -extern uintptr_t _TEXT_START_; -extern uintptr_t _LOAD_START_; -extern uintptr_t _ELF_END_; +extern char _start; +extern char _end; +extern char _ELF_START_; +extern char _TEXT_START_; +extern char _LOAD_START_; +extern char _ELF_END_; bool __libc_initialized = false; From 7a322da6d2fc8bec2e3a569b4d8eb096d6d110cf Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 11 Sep 2018 14:53:33 +0200 Subject: [PATCH 614/723] linux: Solve some ASan positives --- linux/src/drivers/usernet.cpp | 2 +- linux/src/drivers/usernet.hpp | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/linux/src/drivers/usernet.cpp b/linux/src/drivers/usernet.cpp index 83b3a642b6..c4aae521c1 100644 --- a/linux/src/drivers/usernet.cpp +++ b/linux/src/drivers/usernet.cpp @@ -4,7 +4,7 @@ constexpr MAC::Addr UserNet::MAC_ADDRESS; UserNet::UserNet(const uint16_t mtu) - : Link(Link_protocol{{this, &UserNet::transmit}, mac()}), + : Link(Link::Protocol{{this, &UserNet::transmit}, MAC_ADDRESS}), mtu_value(mtu), buffer_store(256u, 128 + mtu) {} UserNet& UserNet::create(const uint16_t mtu) diff --git a/linux/src/drivers/usernet.hpp b/linux/src/drivers/usernet.hpp index 49c94902c1..7fb124127a 100644 --- a/linux/src/drivers/usernet.hpp +++ b/linux/src/drivers/usernet.hpp @@ -25,7 +25,6 @@ class UserNet : public net::Link_layer { public: using Link = net::Link_layer; - using Link_protocol = Link::Protocol; static constexpr MAC::Addr MAC_ADDRESS = {1, 2, 3, 4, 5, 6}; static UserNet& create(const uint16_t MTU); From 9b1fd4341f88230c983fce64cdb628ce03e62ecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 11 Sep 2018 15:28:34 +0200 Subject: [PATCH 615/723] net: Added some extra flags and data to a Conntrack entry --- api/net/conntrack.hpp | 16 ++++++++++++++++ src/net/conntrack.cpp | 12 +++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/api/net/conntrack.hpp b/api/net/conntrack.hpp index 20bfeede5a..6939837d28 100644 --- a/api/net/conntrack.hpp +++ b/api/net/conntrack.hpp @@ -83,6 +83,11 @@ class Conntrack { UNCONFIRMED // not sure about this one }; + enum class Flag : uint8_t { + UNREPLIED = 1 << 0, + ASSURED = 1 << 1 + }; + /** * @brief A entry in the connection tracker (a Connection) */ @@ -92,6 +97,8 @@ class Conntrack { RTC::timestamp_t timeout; Protocol proto; State state; + uint8_t flags{0x0}; + uint8_t other{0x0}; // whoever can make whatever here Entry_handler on_close; Entry(Quadruple quad, Protocol p) @@ -111,6 +118,15 @@ class Conntrack { int deserialize_from(void*); void serialize_to(std::vector&) const; + void set_flag(const Flag f) + { flags |= static_cast(f); } + + void unset_flag(const Flag f) + { flags &= ~static_cast(f); } + + bool isset(const Flag f) const noexcept + { return flags & static_cast(f); } + }; using Timeout_duration = std::chrono::seconds; diff --git a/src/net/conntrack.cpp b/src/net/conntrack.cpp index 7f890dde2a..a8f8d0fa19 100644 --- a/src/net/conntrack.cpp +++ b/src/net/conntrack.cpp @@ -48,10 +48,20 @@ std::string state_str(const Conntrack::State state) } } +std::string flag_str(const uint8_t flags) +{ + std::string str; + if(flags & static_cast(Conntrack::Flag::UNREPLIED)) + str.append(" UNREPLIED"); + if(flags & static_cast(Conntrack::Flag::ASSURED)) + str.append(" ASSURED"); + return str; +} + std::string Conntrack::Entry::to_string() const { return "[ " + first.to_string() + " ] [ " + second.to_string() + " ]" - + " P: " + proto_str(proto) + " S: " + state_str(state); + + " P: " + proto_str(proto) + " S: " + state_str(state) + " F:" + flag_str(flags); } Conntrack::Entry::~Entry() From b215a242eae323748b05700aadc058d842c71df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 11 Sep 2018 15:31:34 +0200 Subject: [PATCH 616/723] net: Added TCP module for Conntrack (supposed to be smarter) --- api/net/tcp/tcp_conntrack.hpp | 52 ++++++ src/CMakeLists.txt | 1 + src/net/tcp/tcp_conntrack.cpp | 202 +++++++++++++++++++++++ test/net/integration/gateway/service.cpp | 7 +- 4 files changed, 261 insertions(+), 1 deletion(-) create mode 100644 api/net/tcp/tcp_conntrack.hpp create mode 100644 src/net/tcp/tcp_conntrack.cpp diff --git a/api/net/tcp/tcp_conntrack.hpp b/api/net/tcp/tcp_conntrack.hpp new file mode 100644 index 0000000000..fc53d877dd --- /dev/null +++ b/api/net/tcp/tcp_conntrack.hpp @@ -0,0 +1,52 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace net::tcp { + + Conntrack::Entry* tcp4_conntrack(Conntrack& ct, Quadruple q, const PacketIP4& pkt); + + enum class Ct_state : uint8_t + { + NONE, + SYN_SENT, + SYN_RECV, + ESTABLISHED, + FIN_WAIT, + TIME_WAIT, + CLOSE_WAIT, + LAST_ACK, + CLOSE + }; + + namespace timeout + { + static constexpr Conntrack::Timeout_duration ESTABLISHED {24*60*60}; + static constexpr Conntrack::Timeout_duration SYN_SENT {2*60}; + static constexpr Conntrack::Timeout_duration SYN_RECV {60}; + static constexpr Conntrack::Timeout_duration FIN_WAIT {2*60}; + static constexpr Conntrack::Timeout_duration TIME_WAIT {2*60}; + static constexpr Conntrack::Timeout_duration CLOSE_WAIT {12*60}; + static constexpr Conntrack::Timeout_duration LAST_ACK {30}; + + static constexpr Conntrack::Timeout_duration NONE {30*60}; + static constexpr Conntrack::Timeout_duration CLOSE {10}; + } + +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2cb929e69f..5fde7f2424 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,6 +50,7 @@ set(OS_OBJECTS net/tcp/tcp.cpp net/tcp/connection.cpp net/tcp/connection_states.cpp net/tcp/write_queue.cpp net/tcp/rttm.cpp net/tcp/listener.cpp net/tcp/read_buffer.cpp net/tcp/read_request.cpp net/tcp/stream.cpp + net/tcp/tcp_conntrack.cpp net/ip4/icmp4.cpp net/ip4/udp.cpp net/ip4/udp_socket.cpp net/ip6/ip6.cpp net/ip6/icmp6.cpp net/ip6/ndp.cpp net/dns/dns.cpp net/dns/client.cpp net/dhcp/dh4client.cpp net/dhcp/dhcpd.cpp diff --git a/src/net/tcp/tcp_conntrack.cpp b/src/net/tcp/tcp_conntrack.cpp new file mode 100644 index 0000000000..9b7fae083a --- /dev/null +++ b/src/net/tcp/tcp_conntrack.cpp @@ -0,0 +1,202 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//#define CT_DEBUG 1 +#ifdef CT_DEBUG +#define CTDBG(fmt, ...) printf(fmt, ##__VA_ARGS__) +#else +#define CTDBG(fmt, ...) /* fmt */ +#endif + +#include +#include + +namespace net::tcp { + + static inline Ct_state state(Conntrack::Entry* entry); + static inline std::string state_str(const uint8_t state); + + static inline void update_timeout(Conntrack::Entry* entry); + static inline void set_state(Conntrack::Entry* entry, const Ct_state state); + static inline void close(Conntrack::Entry* entry); + + Conntrack::Entry* tcp4_conntrack(Conntrack& ct, Quadruple q, const PacketIP4& pkt) + { + using State = Conntrack::State; + const auto proto = Protocol::TCP; + // find the entry + auto* entry = ct.get(q, Protocol::TCP); + + const auto tcp = Packet4_v(const_cast(&pkt)); + CTDBG(" Packet: %s\n", tcp.to_string().c_str()); + + // 1. Never seen before, this is hopefully the SYN + if(UNLIKELY(entry == nullptr)) + { + // first packet should be SYN + if(tcp.isset(SYN)) + { + // this was a SYN, set internal state SYN_SENT & UNREPLIED + entry = ct.add_entry(q, proto); + entry->set_flag(Conntrack::Flag::UNREPLIED); + set_state(entry, Ct_state::SYN_SENT); + + CTDBG(" Entry created: %s State: %s\n", + entry->to_string().c_str(), state_str(entry->other).c_str()); + } + + return entry; // could be nullptr if no SYN + } + + CTDBG(" Entry found: %s State: %s\n", + entry->to_string().c_str(), state_str(entry->other).c_str()); + + // On reset we just have to terminate + if(UNLIKELY(tcp.isset(RST))) + { + close(entry); + return entry; + } + + // 2. This is the SYN/ACK reply + if(UNLIKELY(state(entry) == Ct_state::SYN_SENT and q == entry->second)) + { + // if SYN/ACK reply + if(tcp.isset(SYN) and tcp.isset(ACK)) + { + entry->unset_flag(Conntrack::Flag::UNREPLIED); + entry->state = State::ESTABLISHED; + + // set internal state SYN_RECV & !UNREPLIED + set_state(entry, Ct_state::SYN_RECV); + } + + return entry; + } + + // 3. ACK to SYN/ACK (handshake done) + if(UNLIKELY(state(entry) == Ct_state::SYN_RECV and q == entry->first)) + { + if(tcp.isset(ACK)) + { + // set internal state ESTABLISHED & ASSURED + entry->set_flag(Conntrack::Flag::ASSURED); + set_state(entry, Ct_state::ESTABLISHED); + } + + return entry; + } + + // Closing + if(UNLIKELY(tcp.isset(FIN))) + { + switch(state(entry)) + { + case Ct_state::ESTABLISHED: + set_state(entry, (q == entry->first) ? Ct_state::FIN_WAIT : Ct_state::CLOSE_WAIT); + break; + case Ct_state::FIN_WAIT: + set_state(entry, Ct_state::TIME_WAIT); + break; + case Ct_state::CLOSE_WAIT: + set_state(entry, Ct_state::LAST_ACK); + break; + default: + printf("FIN in state: %s\n", state_str(entry->other).c_str()); + } + return entry; + } + + // Check for ACK when in closing state + if(UNLIKELY(state(entry) == Ct_state::LAST_ACK or state(entry) == Ct_state::TIME_WAIT)) + { + // TODO: Every packet has ACK, this should probably check if the ACK is for our FIN. + // For that we need to know about the TCP state... (sequence number SND.NXT etc..) + if(tcp.isset(ACK)) + { + close(entry); + return entry; + } + } + + update_timeout(entry); + + CTDBG(" Entry handled: %s State: %s\n", + entry->to_string().c_str(), state_str(entry->other).c_str()); + + return entry; + } + + static inline Ct_state state(Conntrack::Entry* entry) + { + return static_cast(entry->other); + } + + static inline void set_state(Conntrack::Entry* entry, const Ct_state state) + { + CTDBG(" State change %s => %s on %s\n", + state_str(entry->other).c_str(), state_str((uint8_t)state).c_str(), entry->to_string().c_str()); + reinterpret_cast(entry->other) = state; + update_timeout(entry); + } + + static inline void close(Conntrack::Entry* entry) + { + CTDBG(" Closing from state %s\n", state_str(entry->other).c_str()); + set_state(entry, Ct_state::CLOSE); + } + + static inline void update_timeout(Conntrack::Entry* entry) + { + const auto& dur = [&]()->const auto& + { + switch(reinterpret_cast(entry->other)) + { + case Ct_state::ESTABLISHED: return timeout::ESTABLISHED; + case Ct_state::SYN_SENT: return timeout::SYN_SENT; + case Ct_state::SYN_RECV: return timeout::SYN_RECV; + case Ct_state::FIN_WAIT: return timeout::FIN_WAIT; + case Ct_state::TIME_WAIT: return timeout::TIME_WAIT; + case Ct_state::CLOSE_WAIT: return timeout::CLOSE_WAIT; + case Ct_state::LAST_ACK: return timeout::LAST_ACK; + case Ct_state::CLOSE: return timeout::CLOSE; + + default: return timeout::NONE; + } + }(); + + entry->timeout = RTC::now() + dur.count(); + } + + static inline std::string state_str(const uint8_t state) + { + switch(static_cast(state)) + { + case Ct_state::SYN_SENT: return "SYN_SENT"; + case Ct_state::SYN_RECV: return "SYN_RECV"; + case Ct_state::ESTABLISHED: return "ESTABLISHED"; + case Ct_state::CLOSE: return "CLOSE"; + case Ct_state::FIN_WAIT: return "FIN_WAIT"; + case Ct_state::TIME_WAIT: return "TIME_WAIT"; + case Ct_state::CLOSE_WAIT: return "CLOSE_WAIT"; + case Ct_state::LAST_ACK: return "LAST_ACK"; + case Ct_state::NONE: return "NONE"; + default: return "???"; + } + } + + +} diff --git a/test/net/integration/gateway/service.cpp b/test/net/integration/gateway/service.cpp index e653d0445f..a63377e048 100644 --- a/test/net/integration/gateway/service.cpp +++ b/test/net/integration/gateway/service.cpp @@ -27,6 +27,8 @@ void verify() { if (++i == 8) printf("SUCCESS\n"); } +#include + void Service::start() { static auto& eth0 = Inet::stack<0>(); @@ -37,6 +39,8 @@ void Service::start() static auto& host2 = Inet::stack<3>(); + eth0.conntrack()->tcp_in = net::tcp::tcp4_conntrack; + INFO("Ping", "host1 => host2 (%s)", host2.ip_addr().to_string().c_str()); host1.icmp().ping(host2.ip_addr(), [](auto reply) { CHECKSERT(reply, "Got pong from %s", host2.ip_addr().to_string().c_str()); @@ -66,7 +70,8 @@ void Service::start() }, 1); INFO("TCP", "host2 => listen:5000"); - host2.tcp().listen(5000, [](auto) {}); + host2.tcp().listen(5000, [](auto /*conn*/) { + }); INFO("TCP", "host1 => host2 (%s:%i)", host2.ip_addr().to_string().c_str(), 5000); host1.tcp().connect({host2.ip_addr(), 5000}, [](auto conn) { From 196e7be9073ac37ed502faecf045c1b761cbfa38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 11 Sep 2018 15:48:17 +0200 Subject: [PATCH 617/723] net: Avoid IP4 accepting bcast not for us #1518 --- src/net/ip4/ip4.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net/ip4/ip4.cpp b/src/net/ip4/ip4.cpp index 458062241a..ede88f7a2a 100644 --- a/src/net/ip4/ip4.cpp +++ b/src/net/ip4/ip4.cpp @@ -102,8 +102,8 @@ namespace net { bool IP4::is_for_me(ip4::Addr dst) const { return stack_.is_valid_source(dst) - or (dst | stack_.netmask()) == ADDR_BCAST - or local_ip() == ADDR_ANY; + or dst == stack_.broadcast_addr() + or dst == ADDR_BCAST; } void IP4::receive(Packet_ptr pckt, [[maybe_unused]]const bool link_bcast) From b5017ffd109eb74bfe5d1fade7376c861b1b146a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 12 Sep 2018 12:09:38 +0200 Subject: [PATCH 618/723] Stdout and driver global construction points --- api/kernel/os.hpp | 6 ++++ src/arch/i686/linker.ld | 16 +++++++++++ src/arch/x86_64/linker.ld | 19 ++++++++++-- src/drivers/CMakeLists.txt | 23 +++++++++------ src/drivers/{ => stdout}/bootlog.cpp | 0 src/drivers/{ => stdout}/default_stdout.cpp | 0 src/drivers/{ => stdout}/timestamps.cpp | 0 src/drivers/{ => stdout}/vgaout.cpp | 0 src/kernel/os.cpp | 22 ++++---------- src/kernel/pci_manager.cpp | 19 +++++------- src/kernel/solo5_manager.cpp | 7 ++--- src/platform/x86_pc/os.cpp | 4 +++ src/platform/x86_pc/platform.cpp | 5 ++++ src/platform/x86_solo5/os.cpp | 32 ++++++++++++--------- 14 files changed, 98 insertions(+), 55 deletions(-) rename src/drivers/{ => stdout}/bootlog.cpp (100%) rename src/drivers/{ => stdout}/default_stdout.cpp (100%) rename src/drivers/{ => stdout}/timestamps.cpp (100%) rename src/drivers/{ => stdout}/vgaout.cpp (100%) diff --git a/api/kernel/os.hpp b/api/kernel/os.hpp index 4ac23afda4..9b63b8d657 100644 --- a/api/kernel/os.hpp +++ b/api/kernel/os.hpp @@ -262,6 +262,12 @@ class OS { static void resume_softreset(intptr_t boot_addr); static void setup_liveupdate(uintptr_t phys = 0); + typedef void (*ctor_t) (); + static void run_ctors(ctor_t* begin, ctor_t* end) + { + for (; begin < end; begin++) (*begin)(); + } + private: /** Process multiboot info. Called by 'start' if multibooted **/ static void multiboot(uint32_t boot_addr); diff --git a/src/arch/i686/linker.ld b/src/arch/i686/linker.ld index 986bb6131a..84ade420e7 100644 --- a/src/arch/i686/linker.ld +++ b/src/arch/i686/linker.ld @@ -73,6 +73,22 @@ SECTIONS PROVIDE_HIDDEN (__init_array_end = .); } + /* Stdout g constructors */ + .stdout_ctors : + { + PROVIDE_HIDDEN (__stdout_ctors_start = .); + */x86_64/drivers/stdout/lib*.a:*(.init_array* .ctors*) + PROVIDE_HIDDEN (__stdout_ctors_end = .); + } + + /* Driver g constructors */ + .driver_ctors : + { + PROVIDE_HIDDEN (__driver_ctors_start = .); + */x86_64/drivers/lib*.a:*(.init_array* .ctors*) + PROVIDE_HIDDEN (__driver_ctors_end = .); + } + /* Plugin constructors */ .plugin_ctors : { diff --git a/src/arch/x86_64/linker.ld b/src/arch/x86_64/linker.ld index 54dcf59d56..344bb97b31 100644 --- a/src/arch/x86_64/linker.ld +++ b/src/arch/x86_64/linker.ld @@ -73,11 +73,26 @@ SECTIONS PROVIDE_HIDDEN (__init_array_start = .); */x86_64/lib/lib*.a:*(.init_array* .ctors*) */x86_64/platform/lib*.a:*(.init_array* .ctors*) - */x86_64/drivers/lib*.a:*(.init_array* .ctors*) PROVIDE_HIDDEN (__init_array_end = .); } - /* Plugin constructors */ + /* Stdout g constructors */ + .stdout_ctors : + { + PROVIDE_HIDDEN (__stdout_ctors_start = .); + */x86_64/drivers/stdout/lib*.a:*(.init_array* .ctors*) + PROVIDE_HIDDEN (__stdout_ctors_end = .); + } + + /* Driver g constructors */ + .driver_ctors : + { + PROVIDE_HIDDEN (__driver_ctors_start = .); + */x86_64/drivers/lib*.a:*(.init_array* .ctors*) + PROVIDE_HIDDEN (__driver_ctors_end = .); + } + + /* Plugin g constructors */ .plugin_ctors : { PROVIDE_HIDDEN (__plugin_ctors_start = .); diff --git a/src/drivers/CMakeLists.txt b/src/drivers/CMakeLists.txt index 1a80731c47..89250da08f 100644 --- a/src/drivers/CMakeLists.txt +++ b/src/drivers/CMakeLists.txt @@ -9,8 +9,8 @@ include_directories(${INCLUDEOS_ROOT}/lib/LiveUpdate) # Simple stuff -add_library(boot_logger STATIC bootlog.cpp) -add_library(default_stdout STATIC "default_stdout.cpp") +add_library(boot_logger STATIC stdout/bootlog.cpp) +add_library(default_stdout STATIC "stdout/default_stdout.cpp") add_library(ide_readwrite STATIC ide.cpp) set_target_properties(ide_readwrite PROPERTIES COMPILE_FLAGS "-DIDE_ENABLE_READ -DIDE_ENABLE_WRITE") @@ -49,10 +49,10 @@ add_dependencies(disk_logger PrecompiledLibraries) add_library(disklog_reader STATIC "disklog_reader.cpp") add_dependencies(disklog_reader PrecompiledLibraries) -add_library(timestamps STATIC timestamps.cpp) +add_library(timestamps STATIC stdout/timestamps.cpp) add_dependencies(timestamps PrecompiledLibraries) -add_library(vga_output STATIC vgaout.cpp) +add_library(vga_output STATIC stdout/vgaout.cpp) add_dependencies(vga_output PrecompiledLibraries) add_library(vga_emergency STATIC vga_emergency.cpp) @@ -63,17 +63,24 @@ add_dependencies(vga_emergency PrecompiledLibraries) # set(CMAKE_INSTALL_MESSAGE LAZY) # to avoid spam +# installation of stdout drivers install(TARGETS default_stdout + boot_logger + timestamps + vga_output + vga_emergency + DESTINATION includeos/${ARCH}/drivers/stdout) + +# installation of drivers (called just before plugins) +install(TARGETS ide_readwrite ide_readonly ide_writeonly virtionet virtioblk virtiocon vmxnet3 e1000 ip4_reassembly heap_debugging - boot_logger disk_logger disklog_reader - timestamps - vga_output vga_emergency - DESTINATION includeos/${ARCH}/drivers) + disk_logger disklog_reader + DESTINATION includeos/${ARCH}/drivers) if(WITH_SOLO5) add_library(solo5blk STATIC solo5blk.cpp) diff --git a/src/drivers/bootlog.cpp b/src/drivers/stdout/bootlog.cpp similarity index 100% rename from src/drivers/bootlog.cpp rename to src/drivers/stdout/bootlog.cpp diff --git a/src/drivers/default_stdout.cpp b/src/drivers/stdout/default_stdout.cpp similarity index 100% rename from src/drivers/default_stdout.cpp rename to src/drivers/stdout/default_stdout.cpp diff --git a/src/drivers/timestamps.cpp b/src/drivers/stdout/timestamps.cpp similarity index 100% rename from src/drivers/timestamps.cpp rename to src/drivers/stdout/timestamps.cpp diff --git a/src/drivers/vgaout.cpp b/src/drivers/stdout/vgaout.cpp similarity index 100% rename from src/drivers/vgaout.cpp rename to src/drivers/stdout/vgaout.cpp diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index d355e2583f..c9731a79a1 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -92,18 +92,10 @@ const char* OS::cmdline_args() noexcept { return cmdline; } -typedef void (*ctor_t) (); -extern ctor_t __service_ctors_start; -extern ctor_t __service_ctors_end; -extern ctor_t __plugin_ctors_start; -extern ctor_t __plugin_ctors_end; - -int __run_ctors(ctor_t* begin, ctor_t* end) -{ - int i = 0; - for (; begin < end; begin++, i++) (*begin)(); - return i; -} +extern OS::ctor_t __plugin_ctors_start; +extern OS::ctor_t __plugin_ctors_end; +extern OS::ctor_t __service_ctors_start; +extern OS::ctor_t __service_ctors_end; void OS::register_plugin(Plugin delg, const char* name){ MYINFO("Registering plugin %s", name); @@ -141,9 +133,7 @@ void OS::post_start() // Custom initialization functions MYINFO("Initializing plugins"); - - // Run plugin constructors - __run_ctors(&__plugin_ctors_start, &__plugin_ctors_end); + OS::run_ctors(&__plugin_ctors_start, &__plugin_ctors_end); // Run plugins PROFILE("Plugins init"); @@ -158,7 +148,7 @@ void OS::post_start() OS::boot_sequence_passed_ = true; // Run service constructors - __run_ctors(&__service_ctors_start, &__service_ctors_end); + OS::run_ctors(&__service_ctors_start, &__service_ctors_end); PROFILE("Service::start"); // begin service start diff --git a/src/kernel/pci_manager.cpp b/src/kernel/pci_manager.cpp index 3042bf1853..1c5ceaaf45 100644 --- a/src/kernel/pci_manager.cpp +++ b/src/kernel/pci_manager.cpp @@ -19,30 +19,25 @@ #include #include #include +#include + #include #include #include -#include - -static const int ELEMENTS = 64; template using Driver_entry = std::pair; template -using fixed_factory_t = Fixed_vector, ELEMENTS>; +using fixed_factory_t = std::vector>; -static // PCI devices -Fixed_vector devices_(Fixedvector_Init::UNINIT); -// driver factories -static -fixed_factory_t nic_fact(Fixedvector_Init::UNINIT); -static -fixed_factory_t blk_fact(Fixedvector_Init::UNINIT); +static std::vector devices_; +static std::vector> nic_fact; +static std::vector> blk_fact; template static inline bool register_device(hw::PCI_Device& dev, fixed_factory_t& factory) { - for (auto& fact : factory) { + for (const auto& fact : factory) { if (fact.first == dev.vendor_product()) { INFO2("| +-o Driver found, initializing "); diff --git a/src/kernel/solo5_manager.cpp b/src/kernel/solo5_manager.cpp index 6d5436de37..140beb3a2c 100644 --- a/src/kernel/solo5_manager.cpp +++ b/src/kernel/solo5_manager.cpp @@ -18,18 +18,17 @@ #include #include #include +#include #include #include -#include -static const int ELEMENTS = 4; using namespace hw; using Nic_ptr = std::unique_ptr; using Blk_ptr = std::unique_ptr; -Fixed_vector, ELEMENTS> nics(Fixedvector_Init::UNINIT); -Fixed_vector, ELEMENTS> blks(Fixedvector_Init::UNINIT); +static std::vector> nics; +static std::vector> blks; void Solo5_manager::register_net(delegate func) { diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index e7369b6776..727a85e1ce 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -91,6 +91,10 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) OS::add_stdout(&OS::default_stdout); } + extern OS::ctor_t __stdout_ctors_start; + extern OS::ctor_t __stdout_ctors_end; + OS::run_ctors(&__stdout_ctors_start, &__stdout_ctors_end); + // Print a fancy header CAPTION("#include // Literally"); diff --git a/src/platform/x86_pc/platform.cpp b/src/platform/x86_pc/platform.cpp index c2d27fe5fd..7eccd566c9 100644 --- a/src/platform/x86_pc/platform.cpp +++ b/src/platform/x86_pc/platform.cpp @@ -93,6 +93,11 @@ void __platform_init() // Deferred call to Service::ready() when calibration is complete x86::APIC_Timer::calibrate(); + INFO2("Initializing drivers"); + extern OS::ctor_t __driver_ctors_start; + extern OS::ctor_t __driver_ctors_end; + OS::run_ctors(&__driver_ctors_start, &__driver_ctors_end); + // Initialize storage devices PCI_manager::init(PCI::STORAGE); OS::m_block_drivers_ready = true; diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index fe1a284cac..3edf936830 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -18,18 +18,13 @@ extern "C" void* get_cpu_esp(); extern uintptr_t _start; extern uintptr_t _end; extern uintptr_t mem_size; -extern uintptr_t _ELF_START_; -extern uintptr_t _TEXT_START_; -extern uintptr_t _LOAD_START_; -extern uintptr_t _ELF_END_; +extern char _ELF_START_; +extern char _TEXT_START_; +extern char _LOAD_START_; +extern char _ELF_END_; // in kernel/os.cpp extern bool os_default_stdout; -typedef void (*ctor_t) (); -extern ctor_t __init_array_start; -extern ctor_t __init_array_end; -extern int __run_ctors(ctor_t* begin, ctor_t* end); - #define MYINFO(X,...) INFO("Kernel", X, ##__VA_ARGS__) #ifdef ENABLE_PROFILERS @@ -81,6 +76,17 @@ void OS::start(const char* cmdline) OS::add_stdout(&OS::default_stdout); } + PROFILE("Global stdout constructors"); + extern OS::ctor_t __stdout_ctors_start; + extern OS::ctor_t __stdout_ctors_end; + OS::run_ctors(&__stdout_ctors_start, &__stdout_ctors_end); + + // Call global ctors + PROFILE("Global kernel constructors"); + extern OS::ctor_t __init_array_start; + extern OS::ctor_t __init_array_end; + OS::run_ctors(&__init_array_start, &__init_array_end); + PROFILE(""); // Print a fancy header CAPTION("#include // Literally"); @@ -92,10 +98,6 @@ void OS::start(const char* cmdline) /// initialize on page 7, 2 pages in size Statman::get().init(0x6000, 0x3000); - // Call global ctors - PROFILE("Global kernel constructors"); - __run_ctors(&__init_array_start, &__init_array_end); - PROFILE("Memory map"); // Assign memory ranges used by the kernel auto& memmap = memory_map(); @@ -129,6 +131,10 @@ void OS::start(const char* cmdline) MYINFO("Booted at monotonic_ns=%ld walltime_ns=%ld", solo5_clock_monotonic(), solo5_clock_wall()); + extern OS::ctor_t __driver_ctors_start; + extern OS::ctor_t __driver_ctors_end; + OS::run_ctors(&__driver_ctors_start, &__driver_ctors_end); + Solo5_manager::init(); // We don't need a start or stop function in solo5. From 5f75ac62a7440bf5421c89f885edec784c0b2459 Mon Sep 17 00:00:00 2001 From: Ingve Vormestrand Date: Wed, 12 Sep 2018 22:50:49 +0200 Subject: [PATCH 619/723] starbase: add timestamps driver by default --- lib/uplink/starbase/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/uplink/starbase/CMakeLists.txt b/lib/uplink/starbase/CMakeLists.txt index 58c5847ed9..f500cd40b5 100644 --- a/lib/uplink/starbase/CMakeLists.txt +++ b/lib/uplink/starbase/CMakeLists.txt @@ -45,6 +45,7 @@ set(DRIVERS vga_output boot_logger e1000 + timestamps ) set(PLUGINS From 8c9a3ca7e2180a3a4875108a4f00119e6deabc37 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 13 Sep 2018 10:33:06 +0200 Subject: [PATCH 620/723] cmake: Add new STDOUT list, more compatibility with old system --- CMakeLists.txt | 6 +++--- cmake/post.service.cmake | 6 +++++- examples/IRCd/CMakeLists.txt | 10 +++++++--- examples/STREAM/CMakeLists.txt | 19 +++++-------------- examples/TCP_perf/CMakeLists.txt | 2 ++ examples/TCP_perf/vm.json | 2 +- examples/snake/CMakeLists.txt | 9 ++++++--- examples/userspace_demo/README.md | 6 ------ examples/websocket/CMakeLists.txt | 22 +++------------------- src/drivers/CMakeLists.txt | 4 ++-- 10 files changed, 34 insertions(+), 52 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2c1c3953c1..0cdbea922a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -99,7 +99,7 @@ endif(minimal) # Set debug options if(debug OR debug-all) - set(CAPABS "${CAPABS} -ggdb3 -DGSL_THROW_ON_CONTRACT_VIOLATION") + set(CAPABS "${CAPABS} -DGSL_THROW_ON_CONTRACT_VIOLATION") endif(debug OR debug-all) if(debug-info OR debug-all) @@ -140,8 +140,8 @@ enable_language(ASM_NASM) # initialize C and C++ compiler flags if (CMAKE_COMPILER_IS_GNUCC) # gcc/g++ settings - set(CMAKE_CXX_FLAGS "${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c") - set(CMAKE_C_FLAGS " ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c") + set(CMAKE_CXX_FLAGS "${CAPABS} -g -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c") + set(CMAKE_C_FLAGS " ${CAPABS} -g ${WARNS} -nostdlib -fno-omit-frame-pointer -c") else() # these kinda work with llvm set(CMAKE_CXX_FLAGS "${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c") diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 202dee242c..55632abcf3 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -101,7 +101,7 @@ endif() # Add default stdout driver if option is ON if (default_stdout) - set(DRIVERS ${DRIVERS} default_stdout) + set(STDOUT ${STDOUT} default_stdout) endif() # Add extra drivers defined from command line @@ -178,18 +178,22 @@ function(plugin_config_option type plugin_list) endfunction() # Location of installed drivers / plugins +set(STDOUT_LOC ${INSTALL_LOC}/${ARCH}/drivers/stdout) set(DRIVER_LOC ${INSTALL_LOC}/${ARCH}/drivers) set(PLUGIN_LOC ${INSTALL_LOC}/${ARCH}/plugins) # Enable DRIVERS which may be specified by parent cmake list +enable_plugins(STDOUT ${STDOUT_LOC}) enable_plugins(DRIVERS ${DRIVER_LOC}) enable_plugins(PLUGINS ${PLUGIN_LOC}) # Global lists of installed Drivers / Plugins +file(GLOB STDOUT_LIST "${STDOUT_LOC}/*.a") file(GLOB DRIVER_LIST "${DRIVER_LOC}/*.a") file(GLOB PLUGIN_LIST "${PLUGIN_LOC}/*.a") # Set configure option for each installed driver +plugin_config_option(stdout STDOUT_LIST) plugin_config_option(driver DRIVER_LIST) plugin_config_option(plugin PLUGIN_LIST) diff --git a/examples/IRCd/CMakeLists.txt b/examples/IRCd/CMakeLists.txt index 2973cfb59a..b60e10b5c0 100644 --- a/examples/IRCd/CMakeLists.txt +++ b/examples/IRCd/CMakeLists.txt @@ -44,18 +44,22 @@ if (REAL_HW) boot_logger e1000 vmxnet3 - vga_output - vga_emergency ) + set(STDOUT + vga_output + vga_emergency + ) else() set(DRIVERS virtionet vmxnet3 - vga_output ) set (PLUGINS uplink ) + set(STDOUT + vga_output + ) endif() set(LIBRARIES libliveupdate.a) diff --git a/examples/STREAM/CMakeLists.txt b/examples/STREAM/CMakeLists.txt index bba79d6def..c14ff69806 100644 --- a/examples/STREAM/CMakeLists.txt +++ b/examples/STREAM/CMakeLists.txt @@ -17,27 +17,18 @@ set(SOURCES service.cpp stream.cpp # ...add more here ) -# To add your own include paths: -# set(LOCAL_INCLUDES ".") - -# Adding memdisk (expects my.disk to exist in current dir): -# set(MEMDISK ${CMAKE_SOURCE_DIR}/my.disk) - # DRIVERS / PLUGINS: - -set(DRIVERS +set(STDOUT vga_output vga_emergency - # virtionet # Virtio networking - # virtioblock # Virtio block device - # ... Others from IncludeOS/src/drivers ) -set(PLUGINS - # syslogd # Syslog over UDP - # ...others +set(DRIVERS + timestamps ) +set(PLUGINS + ) # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/examples/TCP_perf/CMakeLists.txt b/examples/TCP_perf/CMakeLists.txt index 9b3c165586..2ccb25320c 100644 --- a/examples/TCP_perf/CMakeLists.txt +++ b/examples/TCP_perf/CMakeLists.txt @@ -32,5 +32,7 @@ else() ) endif() +#set(STDOUT vga_output) + # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/examples/TCP_perf/vm.json b/examples/TCP_perf/vm.json index e28badab62..fa758ae261 100644 --- a/examples/TCP_perf/vm.json +++ b/examples/TCP_perf/vm.json @@ -1,4 +1,4 @@ { "net" : [{"device" : "vmxnet3"}], - "mem" : 1280 + "mem" : 2048 } diff --git a/examples/snake/CMakeLists.txt b/examples/snake/CMakeLists.txt index 8eaad16660..1bd33148d8 100644 --- a/examples/snake/CMakeLists.txt +++ b/examples/snake/CMakeLists.txt @@ -15,12 +15,15 @@ set(BINARY "snake_example") # Source files to be linked with OS library parts to form bootable image set(SOURCES - service.cpp # ...add more here + service.cpp # ...add more here ) set(DRIVERS - virtionet # Virtio networking - vga_output + virtionet # Virtio networking + ) + +set(STDOUT + vga_output ) # include service build script diff --git a/examples/userspace_demo/README.md b/examples/userspace_demo/README.md index d2a1284121..99b30d2322 100644 --- a/examples/userspace_demo/README.md +++ b/examples/userspace_demo/README.md @@ -1,11 +1,5 @@ ### IncludeOS demo in Linux Userspace -Build and install the linux userspace library first, from the linux folder: -``` -mkdir -p build && pushd build && cmake .. -DCMAKE_INSTALL_PREFIX=$INCLUDEOS_PREFIX && make -j4 install && popd -``` - -Then run: ``` sudo mknod /dev/net/tap c 10 200 lxp-run diff --git a/examples/websocket/CMakeLists.txt b/examples/websocket/CMakeLists.txt index 799056b74c..63882f6e5f 100644 --- a/examples/websocket/CMakeLists.txt +++ b/examples/websocket/CMakeLists.txt @@ -5,12 +5,10 @@ if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) set(ENV{INCLUDEOS_PREFIX} /usr/local) endif() include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) - -project (ws_example) +project(service) # Human-readable name of your service set(SERVICE_NAME "IncludeOS WebSocket Example") - # Name of your service binary set(BINARY "ws_example") @@ -19,28 +17,14 @@ set(SOURCES service.cpp # ...add more here ) -# -# Service CMake options -# (uncomment to enable) -# - -# MISC: - -# To add your own include paths: -# set(LOCAL_INCLUDES ".") - -# Adding memdisk (expects my.disk to exist in current dir): -# set(MEMDISK ${CMAKE_SOURCE_DIR}/my.disk) - -# DRIVERS / PLUGINS: - if ("$ENV{PLATFORM}" STREQUAL "x86_solo5") set(DRIVERS solo5net ) else() set(DRIVERS - virtionet # Virtio networking + virtionet + vmxnet3 ) endif() diff --git a/src/drivers/CMakeLists.txt b/src/drivers/CMakeLists.txt index 89250da08f..93bee7b42b 100644 --- a/src/drivers/CMakeLists.txt +++ b/src/drivers/CMakeLists.txt @@ -66,8 +66,6 @@ set(CMAKE_INSTALL_MESSAGE LAZY) # to avoid spam # installation of stdout drivers install(TARGETS default_stdout - boot_logger - timestamps vga_output vga_emergency DESTINATION includeos/${ARCH}/drivers/stdout) @@ -80,6 +78,8 @@ install(TARGETS ip4_reassembly heap_debugging disk_logger disklog_reader + # stdout drivers that are simple enough to remain here + boot_logger timestamps DESTINATION includeos/${ARCH}/drivers) if(WITH_SOLO5) From b05eafabadcd960708b925568b98579fc95841b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 13 Sep 2018 11:59:46 +0200 Subject: [PATCH 621/723] net: Removed rogue printf in tcp conntrack --- src/net/tcp/tcp_conntrack.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/net/tcp/tcp_conntrack.cpp b/src/net/tcp/tcp_conntrack.cpp index 9b7fae083a..789a158149 100644 --- a/src/net/tcp/tcp_conntrack.cpp +++ b/src/net/tcp/tcp_conntrack.cpp @@ -27,6 +27,7 @@ namespace net::tcp { static inline Ct_state state(Conntrack::Entry* entry); + [[maybe_unused]] static inline std::string state_str(const uint8_t state); static inline void update_timeout(Conntrack::Entry* entry); @@ -115,7 +116,7 @@ namespace net::tcp { set_state(entry, Ct_state::LAST_ACK); break; default: - printf("FIN in state: %s\n", state_str(entry->other).c_str()); + CTDBG("FIN in state: %s\n", state_str(entry->other).c_str()); } return entry; } From eb01a8e54736406776fc12fad09c2a5cd575c269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 13 Sep 2018 12:00:33 +0200 Subject: [PATCH 622/723] test: Simplified DNS integration test --- test/net/integration/dns/CMakeLists.txt | 2 + test/net/integration/dns/service.cpp | 107 ++++++++++++------------ test/net/integration/dns/vm.json | 2 +- 3 files changed, 56 insertions(+), 55 deletions(-) diff --git a/test/net/integration/dns/CMakeLists.txt b/test/net/integration/dns/CMakeLists.txt index f1e04e74c3..5d134c3f87 100644 --- a/test/net/integration/dns/CMakeLists.txt +++ b/test/net/integration/dns/CMakeLists.txt @@ -28,6 +28,8 @@ set(SOURCES set(DRIVERS virtionet + vmxnet3 + e1000 ) # include service build script diff --git a/test/net/integration/dns/service.cpp b/test/net/integration/dns/service.cpp index e863443b1c..928e898c4d 100644 --- a/test/net/integration/dns/service.cpp +++ b/test/net/integration/dns/service.cpp @@ -40,6 +40,40 @@ void print_success(const std::string& hostname, IP4::addr server, IP4::addr res) server.to_string().c_str(), res.to_string().c_str()); } +struct Name_request +{ + std::string name; + ip4::Addr server; + + Name_request(std::string n) + : name{std::move(n)} {} + Name_request(std::string n, ip4::Addr serv) + : name{std::move(n)}, server{serv} {} +}; + +static void do_test(net::Inet& inet, std::vector& reqs) +{ + for(auto& req : reqs) + { + if(req.server == 0) + req.server = inet.dns_addr(); + + inet.resolve(req.name, req.server, + [name = req.name, server = req.server] (ip4::Addr res, const Error& err) + { + if (err) { + print_error(name, server, err); + } + else { + if (res != 0) + print_success(name, server, res); + else + print_not_resolved(name); + } + }); + } +} + void Service::start(const std::string&) { auto& inet = net::Inet::stack<0>(); @@ -47,62 +81,27 @@ void Service::start(const std::string&) { 10, 0, 0, 48 }, // IP { 255, 255, 255, 0 }, // Netmask { 10, 0, 0, 1 }, // Gateway - { 8, 8, 8, 8 } // DNS + { 1, 1, 1, 1 } // DNS ); - const std::string google = "google.com"; - const std::string github = "github.com"; - const std::string guardian = "theguardian.com"; - const std::string some_address = "some_address_that_doesnt_exist.com"; - - const IP4::addr stack_dns = inet.dns_addr(); - const IP4::addr level3 = IP4::addr{4, 2, 2, 1}; - - inet.resolve(google, [google, stack_dns] (IP4::addr res, const Error& err) { - if (err) { - print_error(google, stack_dns, err); - } - else { - if (res != IP4::ADDR_ANY) - print_success(google, stack_dns, res); - else - print_not_resolved(google); - } - }); - - inet.resolve(github, [github, stack_dns] (IP4::addr res, const Error& err) { - if (err) { - print_error(github, stack_dns, err); - } - else { - if (res != IP4::ADDR_ANY) - print_success(github, stack_dns, res); - else - print_not_resolved(github); - } - }); + const ip4::Addr level3{4, 2, 2, 1}; + const ip4::Addr google{8, 8, 8, 8}; - inet.resolve(guardian, level3, [guardian, level3] (IP4::addr res, const Error& err) { - if (err) { - print_error(guardian, level3, err); - } - else { - if (res != IP4::ADDR_ANY) - print_success(guardian, level3, res); - else - print_not_resolved(guardian); - } - }); + static std::vector requests { + {"google.com", google}, + {"github.com", google}, + {"some_address_that_doesnt_exist.com"}, + {"theguardian.com", level3}, + {"www.facebook.com"}, + {"rs.dns-oarc.net"}, + {"reddit.com"}, + {"includeos.io"}, + {"includeos.org"}, + {"doubleclick.net"}, + {"google-analytics.com"}, + {"akamaihd.net"}, + {"googlesyndication.com"} + }; - inet.resolve(some_address, [some_address, stack_dns] (IP4::addr res, const Error& err) { - if (err) { - print_error(some_address, stack_dns, err); - } - else { - if (res != IP4::ADDR_ANY) - print_success(some_address, stack_dns, res); - else - print_not_resolved(some_address); - } - }); + do_test(inet, requests); } diff --git a/test/net/integration/dns/vm.json b/test/net/integration/dns/vm.json index 93dab698fb..97df0a392e 100644 --- a/test/net/integration/dns/vm.json +++ b/test/net/integration/dns/vm.json @@ -1,6 +1,6 @@ { "image" : "test_dns.img", - "net" : [{"device" : "virtio", "mac" : "c0:01:0a:00:00:2a"}], + "net" : [{"device" : "vmxnet3", "mac" : "c0:01:0a:00:00:2a"}], "mem" : 128, "intrusive" : "True" } From 43a11da4bdacac143cf3bed5ce677c8a8fbc02b5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 14 Sep 2018 10:28:59 +0200 Subject: [PATCH 623/723] test: Add heap verification in STL test --- test/stl/integration/stl/CMakeLists.txt | 10 ++--- test/stl/integration/stl/service.cpp | 58 ++++++++++++++++++++++--- test/stl/integration/stl/test.sh | 5 --- 3 files changed, 54 insertions(+), 19 deletions(-) delete mode 100755 test/stl/integration/stl/test.sh diff --git a/test/stl/integration/stl/CMakeLists.txt b/test/stl/integration/stl/CMakeLists.txt index 7888353f3b..f2074b320f 100644 --- a/test/stl/integration/stl/CMakeLists.txt +++ b/test/stl/integration/stl/CMakeLists.txt @@ -1,21 +1,17 @@ - cmake_minimum_required(VERSION 2.8.9) - +cmake_minimum_required(VERSION 2.8.9) if (NOT DEFINED ENV{INCLUDEOS_PREFIX}) set(ENV{INCLUDEOS_PREFIX} /usr/local) endif() - include($ENV{INCLUDEOS_PREFIX}/includeos/pre.service.cmake) - -project (test_STL) +project(service) set(SERVICE_NAME "STL test") set(BINARY "test_STL") -set(MAX_MEM 128) set(SOURCES service.cpp ) -#set(DRIVERS virtionet) +set(PLUGINS vfs) # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/stl/integration/stl/service.cpp b/test/stl/integration/stl/service.cpp index 9f5536576b..302f04e034 100644 --- a/test/stl/integration/stl/service.cpp +++ b/test/stl/integration/stl/service.cpp @@ -27,6 +27,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -90,18 +93,59 @@ const lest::test specification[] = } }; -#define MYINFO(X,...) INFO("Test STL",X,##__VA_ARGS__) -void Service::start(const std::string&) +static void verify_heap_works() { - // Wonder when these are used...? - std::set_terminate([](){ printf("CUSTOM TERMINATE Handler \n"); }); - std::set_new_handler([](){ printf("CUSTOM NEW Handler \n"); }); + static std::array allocs; + static std::array order; + static const int ROUNDS = 2000; + INFO("Heap", "Testing heap allocations"); + for (size_t i = 0; i < order.size(); i++) order[i] = i; + std::random_device rd; + std::mt19937 g(rd()); - // TODO: find some implementation for long double, or not... or use double - //auto sine = sinl(42); + INFO2("Allocating %zu times in %d iterations", allocs.size(), ROUNDS); + for (int i = 0; i < ROUNDS; i++) + { + for (size_t j = 0; j < allocs.size(); j++) + { + const size_t bytes = 1 << (3 + j * 10 / allocs.size()); + try { + allocs[j] = new char[bytes]; + } + catch (const std::exception& e) { + fprintf(stderr, "Allocation failed on run %d for %zu bytes\n", + i, bytes); + assert(0 && "Failed allocation"); + } + } + std::shuffle(order.begin(), order.end(), g); + for (const int index : order) + { + delete[] allocs[index]; + allocs[index] = nullptr; + } + } +} + +#define MYINFO(X,...) INFO("Test STL",X,##__VA_ARGS__) + +void Service::start(const std::string&) +{ + // these will get called when something bad happens + // and should not return + std::set_terminate([]() { + printf("CUSTOM TERMINATE Handler \n"); + std::abort(); + }); + std::set_new_handler([]() { + printf("CUSTOM NEW Handler \n"); + std::abort(); + }); + // test Heap + verify_heap_works(); printf("*** Testing STL Basics - must be verified from the outside ***\n"); MYINFO("The following two lines should be identical, including newline"); diff --git a/test/stl/integration/stl/test.sh b/test/stl/integration/stl/test.sh deleted file mode 100755 index 14e5d77c81..0000000000 --- a/test/stl/integration/stl/test.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -source ../test_base - -make -start test_STL.img From 1ef74c61eac496a2a79829c11a8a4ffe6da2b4f1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 14 Sep 2018 11:07:23 +0200 Subject: [PATCH 624/723] cmake: Simplify debugging option --- CMakeLists.txt | 19 ++++--------------- cmake/post.service.cmake | 8 +++++--- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0cdbea922a..0faabd6b17 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,7 +59,7 @@ option (full_lto "Enable full LTO (compatibility)" OFF) # create random hex string as stack protector canary string(RANDOM LENGTH 16 ALPHABET 0123456789ABCDEF STACK_PROTECTOR_VALUE) -set(CAPABS "${CAPABS} -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE}") +set(CAPABS "${CAPABS} -g -fstack-protector-strong -D_STACK_GUARD_VALUE_=0x${STACK_PROTECTOR_VALUE}") # Various global defines # * NO_DEBUG disables output from the debug macro @@ -68,9 +68,7 @@ set(CAPABS "${CAPABS} -DNO_DEBUG=1 -DOS_TERMINATE_ON_CONTRACT_VIOLATION -D_LIBCP set(WARNS "-Wall -Wextra") # -Werror # configure options -option(debug "Build with debugging symbols (OBS: Dramatically increases binary size)" OFF) -option(debug-info "Build like \"all\" but with debugging output (i.e. the 'debug'-macro) enabled" OFF) -option(debug-all "Build with debugging symbols + debugging output, i.e. \"debug\" + \"debug-info\"" OFF) +option(debug "Build with no optimizations" OFF) option(minimal "Build for minimal size" OFF) option(stripped "reduce size" OFF) @@ -97,15 +95,6 @@ if(minimal) set(OPTIMIZE "-Os") endif(minimal) -# Set debug options -if(debug OR debug-all) - set(CAPABS "${CAPABS} -DGSL_THROW_ON_CONTRACT_VIOLATION") -endif(debug OR debug-all) - -if(debug-info OR debug-all) - set(CAPABS "${CAPABS} -UNO_DEBUG") -endif(debug-info OR debug-all) - if(silent) set(CAPABS "${CAPABS} -DNO-INFO=1") endif(silent) @@ -140,8 +129,8 @@ enable_language(ASM_NASM) # initialize C and C++ compiler flags if (CMAKE_COMPILER_IS_GNUCC) # gcc/g++ settings - set(CMAKE_CXX_FLAGS "${CAPABS} -g -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c") - set(CMAKE_C_FLAGS " ${CAPABS} -g ${WARNS} -nostdlib -fno-omit-frame-pointer -c") + set(CMAKE_CXX_FLAGS "${CAPABS} -std=${CPP_VERSION} ${WARNS} -Wno-frame-address -nostdlib -fno-omit-frame-pointer -c") + set(CMAKE_C_FLAGS " ${CAPABS} ${WARNS} -nostdlib -fno-omit-frame-pointer -c") else() # these kinda work with llvm set(CMAKE_CXX_FLAGS "${CAPABS} -std=${CPP_VERSION} ${WARNS} -nostdlib -nostdlibinc -fno-omit-frame-pointer -c") diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 55632abcf3..7fac66a379 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -488,16 +488,18 @@ target_link_libraries(service # write binary location to known file file(WRITE ${CMAKE_BINARY_DIR}/binary.txt ${BINARY}) -# old behavior: remote all symbols after elfsym +# old behavior: remove all symbols after elfsym if (NOT debug) - set(STRIP_LV ${CMAKE_STRIP} --strip-all ${BINARY}) + set(STRIP_LV ${CMAKE_STRIP} --strip-debug ${BINARY}) +else() + set(STRIP_LV true) endif() add_custom_target( pruned_elf_symbols ALL COMMAND ${INSTALL_LOC}/bin/elf_syms ${BINARY} COMMAND ${CMAKE_OBJCOPY} --update-section .elf_symbols=_elf_symbols.bin ${BINARY} ${BINARY} - #COMMAND ${STRIP_LV} + COMMAND ${STRIP_LV} DEPENDS service ) From 5e0105bb4b155a8ad663e4b578744b67a2e932b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 14 Sep 2018 14:27:25 +0200 Subject: [PATCH 625/723] net: Avoid parsing DNS AAAA records as names --- api/net/dns/dns.hpp | 1 + src/net/dns/dns.cpp | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/api/net/dns/dns.hpp b/api/net/dns/dns.hpp index f7eabf36a1..b75bdc438f 100644 --- a/api/net/dns/dns.hpp +++ b/api/net/dns/dns.hpp @@ -64,6 +64,7 @@ namespace net #define DNS_CLASS_INET 1 #define DNS_TYPE_A 1 // A record +#define DNS_TYPE_AAAA 28 // AAAA record #define DNS_TYPE_NS 2 // respect mah authoritah #define DNS_TYPE_ALIAS 5 // name alias diff --git a/src/net/dns/dns.cpp b/src/net/dns/dns.cpp index c887fc552f..7b5047b1b5 100644 --- a/src/net/dns/dns.cpp +++ b/src/net/dns/dns.cpp @@ -269,6 +269,12 @@ namespace net this->rdata = std::string(reader, len); reader += len; } + else if (ntohs(resource.type) == DNS_TYPE_AAAA) + { + // skip IPv6 records for now + int len = ntohs(resource.data_len); + reader += len; + } else { this->rdata = readName(reader, buffer, stop); From cb0b30c8eaa90821e2985eea24f6993cecf79fcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Fri, 14 Sep 2018 15:20:08 +0200 Subject: [PATCH 626/723] net: Some DNS bandaid to avoid reading names outside buffer --- api/net/dns/dns.hpp | 6 +++--- src/net/dns/client.cpp | 7 +++++-- src/net/dns/dns.cpp | 42 ++++++++++++++++++++++++++++-------------- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/api/net/dns/dns.hpp b/api/net/dns/dns.hpp index b75bdc438f..249fd738b3 100644 --- a/api/net/dns/dns.hpp +++ b/api/net/dns/dns.hpp @@ -151,7 +151,7 @@ namespace net public: using id_t = unsigned short; int create(char* buffer, const std::string& hostname); - bool parseResponse(const char* buffer); + bool parseResponse(const char* buffer, size_t len); void print(const char* buffer) const; const std::string& hostname() const @@ -176,7 +176,7 @@ namespace net private: struct rr_t // resource record { - rr_t(const char*& reader, const char* buffer); + rr_t(const char*& reader, const char* buffer, size_t len); std::string name; std::string rdata; @@ -190,7 +190,7 @@ namespace net private: // decompress names in 3www6google3com format - std::string readName(const char* reader, const char* buffer, int& count); + std::string readName(const char* reader, const char* buffer, size_t len, int& count); }; unsigned short generateID() diff --git a/src/net/dns/client.cpp b/src/net/dns/client.cpp index 6a580add59..d0603029ba 100644 --- a/src/net/dns/client.cpp +++ b/src/net/dns/client.cpp @@ -98,8 +98,11 @@ namespace net flush_timer_.stop(); } - void DNSClient::receive_response(Address, UDP::port_t, const char* data, size_t) + void DNSClient::receive_response(Address, UDP::port_t, const char* data, size_t len) { + if(UNLIKELY(len < sizeof(DNS::header))) + return; // no point in even bothering + const auto& reply = *(DNS::header*) data; // match the transactions id on the reply with the ones in our map auto it = requests_.find(ntohs(reply.id)); @@ -111,7 +114,7 @@ namespace net auto& dns_req = req.request; // parse request - dns_req.parseResponse(data); + dns_req.parseResponse(data, len); // cache the response for 60 seconds if(cache_ttl_ > std::chrono::seconds::zero()) diff --git a/src/net/dns/dns.cpp b/src/net/dns/dns.cpp index 7b5047b1b5..5f9c6bf5a0 100644 --- a/src/net/dns/dns.cpp +++ b/src/net/dns/dns.cpp @@ -179,8 +179,10 @@ namespace net } // parse received message (as put into buffer) - bool DNS::Request::parseResponse(const char* buffer) + bool DNS::Request::parseResponse(const char* buffer, size_t len) { + Expects(len >= sizeof(DNS::header)); + const header* dns = (const header*) buffer; // move ahead of the dns header and the query field @@ -189,17 +191,20 @@ namespace net // .. and past the original question reader += sizeof(DNS::question); + if(UNLIKELY(reader > (buffer + len))) + return false; + // parse answers for(int i = 0; i < ntohs(dns->ans_count); i++) - answers.emplace_back(reader, buffer); + answers.emplace_back(reader, buffer, len); // parse authorities for (int i = 0; i < ntohs(dns->auth_count); i++) - auth.emplace_back(reader, buffer); + auth.emplace_back(reader, buffer, len); // parse additional for (int i = 0; i < ntohs(dns->add_count); i++) - addit.emplace_back(reader, buffer); + addit.emplace_back(reader, buffer, len); return true; } @@ -251,11 +256,11 @@ namespace net *dns++ = '\0'; } - DNS::Request::rr_t::rr_t(const char*& reader, const char* buffer) + DNS::Request::rr_t::rr_t(const char*& reader, const char* buffer, size_t len) { int stop; - this->name = readName(reader, buffer, stop); + this->name = readName(reader, buffer, len, stop); reader += stop; this->resource = *(rr_data*) reader; @@ -264,20 +269,20 @@ namespace net // if its an ipv4 address if (ntohs(resource.type) == DNS_TYPE_A) { - int len = ntohs(resource.data_len); + int dlen = ntohs(resource.data_len); - this->rdata = std::string(reader, len); + this->rdata = std::string(reader, dlen); reader += len; } else if (ntohs(resource.type) == DNS_TYPE_AAAA) { // skip IPv6 records for now - int len = ntohs(resource.data_len); - reader += len; + int dlen = ntohs(resource.data_len); + reader += dlen; } else { - this->rdata = readName(reader, buffer, stop); + this->rdata = readName(reader, buffer, len, stop); reader += stop; } } @@ -317,7 +322,7 @@ namespace net printf("\n"); } - std::string DNS::Request::rr_t::readName(const char* reader, const char* buffer, int& count) + std::string DNS::Request::rr_t::readName(const char* reader, const char* buffer, size_t tot_len, int& count) { std::string name(256, '\0'); unsigned p = 0; @@ -331,7 +336,12 @@ namespace net { if (*ureader >= 192) { - offset = (*ureader) * 256 + *(ureader+1) - 49152; // = 11000000 00000000 + // read 16-bit offset, mask out the 2 top bits + offset = ((*ureader) * 256 + *(ureader+1)) & 0x3FFF; // = 11000000 00000000 + + if(UNLIKELY(offset > tot_len)) + return {}; + ureader = (unsigned char*) buffer + offset - 1; jumped = true; // we have jumped to another location so counting wont go up! } @@ -344,6 +354,11 @@ namespace net // if we havent jumped to another location then we can count up if (jumped == false) count++; } + + // maximum label size + if(UNLIKELY(p > 63)) + return {}; + name.resize(p); // number of steps we actually moved forward in the packet @@ -356,7 +371,6 @@ namespace net for(i = 0; i < len; i++) { p = name[i]; - for(unsigned j = 0; j < p; j++) { name[i] = name[i+1]; From 7fa03da5120304fa05389ce8774455facc9ae325 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Fri, 14 Sep 2018 11:27:07 +0200 Subject: [PATCH 627/723] test: Remove erroneous call to fgetc() after closing file --- test/posix/integration/file_fd/test_file_fd.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/test/posix/integration/file_fd/test_file_fd.cpp b/test/posix/integration/file_fd/test_file_fd.cpp index 77d2a08854..51bdb87253 100644 --- a/test/posix/integration/file_fd/test_file_fd.cpp +++ b/test/posix/integration/file_fd/test_file_fd.cpp @@ -247,9 +247,6 @@ const lest::test specification[] = EXPECT(c != EOF); int res = fclose(f); EXPECT(res == 0); - // try to read another character - c = fgetc(f); - EXPECT(c == EOF); } }, { @@ -257,10 +254,10 @@ const lest::test specification[] = FILE* f = fopen("/mnt/disk/file3", "r"); EXPECT(f != nullptr); char buf[256]; - memset(buf, 0, 256); - char* ptr = fgets(buf, 256, f); + memset(buf, 0, sizeof(buf)); + const char* ptr = fgets(buf, sizeof(buf), f); EXPECT(ptr != nullptr); - int cmp = strcmp(buf, "even more content\n"); + int cmp = strncmp(buf, "even more content\n", sizeof(buf)); EXPECT(cmp == 0); int res = fclose(f); EXPECT(res == 0); @@ -276,7 +273,6 @@ const lest::test specification[] = size_t wanted_count {7}; size_t count = fread(buf, sizeof(char), wanted_count, f); EXPECT(count == wanted_count); - printf("fread result: %zu\n", count); int cmp = strcmp(buf, "content"); EXPECT(cmp == 0); int res = fclose(f); From 43d0eafcf924134d1c6184cd89fed8b32523d5ef Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Sun, 16 Sep 2018 22:15:37 +0200 Subject: [PATCH 628/723] Starbase: Use new STDOUT for vga_output --- lib/uplink/starbase/CMakeLists.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/uplink/starbase/CMakeLists.txt b/lib/uplink/starbase/CMakeLists.txt index f500cd40b5..8b20e1f5ae 100644 --- a/lib/uplink/starbase/CMakeLists.txt +++ b/lib/uplink/starbase/CMakeLists.txt @@ -42,7 +42,6 @@ set(DRIVERS virtionet vmxnet3 uplink_log - vga_output boot_logger e1000 timestamps @@ -54,6 +53,10 @@ set(PLUGINS system_log ) +set(STDOUT + vga_output + ) + # THIRD PARTY LIBRARIES: set(LIBRARIES From 50d1fe77b9c5830e3aa6af536cf08a5b25b084ea Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Sun, 16 Sep 2018 22:23:52 +0200 Subject: [PATCH 629/723] Renamed build_examples to build_services --- test/misc/{build_examples => build_services}/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename test/misc/{build_examples => build_services}/test.sh (92%) diff --git a/test/misc/build_examples/test.sh b/test/misc/build_services/test.sh similarity index 92% rename from test/misc/build_examples/test.sh rename to test/misc/build_services/test.sh index 7dc22e6165..224d27ec31 100755 --- a/test/misc/build_examples/test.sh +++ b/test/misc/build_services/test.sh @@ -56,7 +56,7 @@ function build_service() { export -f build_service -for dir in `ls -d $script_absolute_dir/../../../examples/*` +for dir in `ls -d $script_absolute_dir/../../../examples/* $script_absolute_dir/../../../lib/uplink/starbase` do if [[ $dir == *"$skip_tests"* ]]; then continue From 4928f5ac303c8bc694d308d230b795c1a8077ad3 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Sun, 16 Sep 2018 22:25:54 +0200 Subject: [PATCH 630/723] Test: Added starbase to build_services test --- test/misc/build_services/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/misc/build_services/test.sh b/test/misc/build_services/test.sh index 224d27ec31..ac4713eda3 100755 --- a/test/misc/build_services/test.sh +++ b/test/misc/build_services/test.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Script that builds all the files in the example folder +# Script that builds a number of services # Will print the output if the test fails set -eE From 50b3f46853f30a20d71adc0e21a641dcbd4bffa2 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 17 Sep 2018 10:48:27 +0200 Subject: [PATCH 631/723] diskbuilder: Use lstat() on each entry, avoid using d_type --- diskimagebuild/filetree.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/diskimagebuild/filetree.cpp b/diskimagebuild/filetree.cpp index 3e6f365494..a77115ef75 100644 --- a/diskimagebuild/filetree.cpp +++ b/diskimagebuild/filetree.cpp @@ -109,12 +109,23 @@ void FileSys::add_dir(Dir& dvec) std::string name(ent->d_name); if (name == ".." || name == ".") continue; - if (ent->d_type == DT_DIR) { + struct stat buf; + int res = lstat(ent->d_name, &buf); + if (res < 0) { + fprintf(stderr, "Stat failed on %s with error %s\n", + ent->d_name, strerror(errno)); + continue; + } + + if (S_ISDIR(buf.st_mode)) { sub_dirs.push_back(std::move(name)); } - else { + else if (S_ISREG(buf.st_mode)) { sub_files.push_back(std::move(name)); } + else { + fprintf(stderr, "Encountered unknown entry %s\n", ent->d_name); + } } // close directory before adding more folders and files res = closedir(dir); From f518a479e16c98cf5a8c0023312d7d03736b754f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 18 Sep 2018 10:03:31 +0200 Subject: [PATCH 632/723] cmake: Work towards LTO on GCC --- cmake/post.service.cmake | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 7fac66a379..73bf90329f 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -56,12 +56,28 @@ endif() if (undefined_san) set(CAPABS "${CAPABS} -fsanitize=undefined -fno-sanitize=vptr") endif() -if (thin_lto) - set(CMAKE_LINKER "ld.lld") - set(OPTIMIZE "${OPTIMIZE} -flto=thin -fuse-ld=lld") -elseif (full_lto) - set(CMAKE_LINKER "ld.lld") - set(OPTIMIZE "${OPTIMIZE} -flto=full") + +if (thin_lto OR full_lto) + if (thin_lto) + set(OPTIMIZE "${OPTIMIZE} -flto=thin -fuse-ld=lld") + elseif (full_lto) + set(OPTIMIZE "${OPTIMIZE} -flto=full") + endif() + if (CMAKE_COMPILER_IS_CLANG) + set(CMAKE_LINKER "ld.lld") + elseif (CMAKE_COMPILER_IS_GNUCC) + execute_process(COMMAND ${CMAKE_C_COMPILER} --print-file-name liblto_plugin.so OUTPUT_VARIABLE LTO_PLUGiN OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${CMAKE_C_COMPILER} --print-file-name lto-wrapper OUTPUT_VARIABLE LTO_WRAPPER OUTPUT_STRIP_TRAILING_WHITESPACE) + execute_process(COMMAND ${CMAKE_C_COMPILER} --print-file-name collect2 OUTPUT_VARIABLE GCC_COLLECT OUTPUT_STRIP_TRAILING_WHITESPACE) + message(STATUS "Linker plugin: ${LTO_PLUGiN}, wrapper: ${LTO_WRAPPER}") + message(STATUS "LTO wrapper: ${LTO_WRAPPER}") + set(ENV{COLLECT_GCC} "${GCC_COLLECT}") + set(ENV{COLLECT_LTO_WRAPPER} "${LTO_WRAPPER}") + message(STATUS "COLLECT_GCC=$ENV{COLLECT_GCC}") + set(CMAKE_LINKER "gold") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -plugin ${LTO_PLUGiN} -plugin-opt ${LTO_WRAPPER} -plugin-opt -fresolution=40") + message(STATUS "Linker is now: ${CMAKE_LINKER}") + endif() endif() if (CMAKE_COMPILER_IS_GNUCC) @@ -248,7 +264,7 @@ set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS) # this removed -rdynamic from linker ou set(CMAKE_CXX_LINK_EXECUTABLE " -o ") set(BUILD_SHARED_LIBRARIES OFF) -set(CMAKE_EXE_LINKER_FLAGS "-static") +set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static") set(LD_STRIP) if (NOT debug) From 2165afa7e3ebdea71ad81fcd01b181d857582143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 18 Sep 2018 14:18:15 +0200 Subject: [PATCH 633/723] tcp: Set PSH flag on rtx with data --- src/net/tcp/connection.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/net/tcp/connection.cpp b/src/net/tcp/connection.cpp index dc0d51857d..d2abdaa3a4 100644 --- a/src/net/tcp/connection.cpp +++ b/src/net/tcp/connection.cpp @@ -884,6 +884,7 @@ void Connection::retransmit() { debug2(" With data (wq.sz=%u) buf.unacked=%u\n", writeq.size(), buf.length() - buf.acknowledged); fill_packet(*packet, buf->data() + writeq.acked(), buf->size() - writeq.acked()); + packet->set_flag(PSH); } rtx_attempt_++; packet->set_seq(cb.SND.UNA); From 86a2231986b9d60784c3c54c0905fd592f76ff06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 18 Sep 2018 14:19:02 +0200 Subject: [PATCH 634/723] net: Deserialize new Conntrack information --- src/net/conntrack.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/net/conntrack.cpp b/src/net/conntrack.cpp index a8f8d0fa19..3621f3ef1b 100644 --- a/src/net/conntrack.cpp +++ b/src/net/conntrack.cpp @@ -333,6 +333,8 @@ int Conntrack::Entry::deserialize_from(void* addr) this->timeout = entry.timeout; this->proto = entry.proto; this->state = entry.state; + this->flags = entry.flags; + this->other = entry.other; return sizeof(Entry) - sizeof(on_close); } From 5cef9d1d9b8796843836d716c4da183a75b43447 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 18 Sep 2018 14:25:00 +0200 Subject: [PATCH 635/723] linux: Re-arrange OS/arch functions --- cmake/linux.service.cmake | 7 +++++-- linux/src/arch.cpp | 44 ++++++++++++++++++++++++++++++--------- linux/src/os.cpp | 41 ++++++++---------------------------- 3 files changed, 48 insertions(+), 44 deletions(-) diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index 6705ce81f4..63d1ed0e04 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -106,15 +106,18 @@ if (CUSTOM_BOTAN) target_link_libraries(service ${BOTAN_LIBS} -ldl -pthread) endif() target_link_libraries(service ${PLUGINS_LIST}) -target_link_libraries(service includeos linuxrt includeos http_parser rt) +target_link_libraries(service includeos linuxrt includeos linuxrt http_parser rt) target_link_libraries(service ${EXTRA_LIBS}) if (CUSTOM_BOTAN) target_link_libraries(service ${BOTAN_LIBS}) endif() if (STATIC_BUILD) - target_link_libraries(service -static-libstdc++ -static-libgcc) + set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") + set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") + target_link_libraries(service -static-libstdc++ -static-libgcc -static) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -pthread") + set(BUILD_SHARED_LIBRARIES OFF) endif() if (ENABLE_LTO OR USE_LLD) diff --git a/linux/src/arch.cpp b/linux/src/arch.cpp index 987e75a968..815f6c60c7 100644 --- a/linux/src/arch.cpp +++ b/linux/src/arch.cpp @@ -15,16 +15,6 @@ weak_alias(__service_ctors_start, __service_ctors_end); char _ELF_START_; char _ELF_END_; -uintptr_t OS::heap_max() noexcept -{ - return (uintptr_t) -1; -} - -bool OS::is_panicking() noexcept -{ - return false; -} - void __arch_subscribe_irq(uint8_t) {} uint64_t __arch_system_time() noexcept @@ -44,3 +34,37 @@ void __arch_reboot() { exit(0); } + +#include +void print_backtrace() +{ + static const int NUM_ADDRS = 64; + void* addresses[NUM_ADDRS]; + + int nptrs = backtrace(addresses, NUM_ADDRS); + printf("backtrace() returned %d addresses\n", nptrs); + + /* The call backtrace_symbols_fd(buffer, nptrs, STDOUT_FILENO) + would produce similar output to the following: */ + + char** strings = backtrace_symbols(addresses, nptrs); + if (strings == NULL) { + perror("backtrace_symbols"); + exit(EXIT_FAILURE); + } + + for (int j = 0; j < nptrs; j++) + printf("#%02d: %8p %s\n", j, addresses[j], strings[j]); + + free(strings); +} + +#include +extern "C" +void panic(const char* why) +{ + printf("!! PANIC !!\nReason: %s\n", why); + print_backtrace(); + raise(SIGINT); + exit(1); +} diff --git a/linux/src/os.cpp b/linux/src/os.cpp index 8708ff710c..8ef2081ba2 100644 --- a/linux/src/os.cpp +++ b/linux/src/os.cpp @@ -34,6 +34,15 @@ uintptr_t OS::heap_usage() noexcept { auto info = mallinfo(); return info.arena + info.hblkhd; } +uintptr_t OS::heap_max() noexcept +{ + return (uintptr_t) -1; +} + +bool OS::is_panicking() noexcept +{ + return false; +} #include #include @@ -126,35 +135,3 @@ void* aligned_alloc(size_t alignment, size_t size) { return memalign(alignment, size); } #endif - -#include -void print_backtrace() -{ - static const int NUM_ADDRS = 64; - void* addresses[NUM_ADDRS]; - - int nptrs = backtrace(addresses, NUM_ADDRS); - printf("backtrace() returned %d addresses\n", nptrs); - - /* The call backtrace_symbols_fd(buffer, nptrs, STDOUT_FILENO) - would produce similar output to the following: */ - - char** strings = backtrace_symbols(addresses, nptrs); - if (strings == NULL) { - perror("backtrace_symbols"); - exit(EXIT_FAILURE); - } - - for (int j = 0; j < nptrs; j++) - printf("#%02d: %8p %s\n", j, addresses[j], strings[j]); - - free(strings); -} - -extern "C" -void panic(const char* why) -{ - printf("!! PANIC !!\nReason: %s\n", why); - raise(SIGINT); - exit(1); -} From 6fa5bfeb70432de779277970fe89688c7faac4c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 18 Sep 2018 15:26:41 +0200 Subject: [PATCH 636/723] ip4: Add drop stat for all chains --- api/net/ip4/ip4.hpp | 4 ++++ src/net/ip4/ip4.cpp | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/api/net/ip4/ip4.hpp b/api/net/ip4/ip4.hpp index af34a82276..625df96702 100644 --- a/api/net/ip4/ip4.hpp +++ b/api/net/ip4/ip4.hpp @@ -477,6 +477,10 @@ namespace net { Filter_chain postrouting_chain_{"Postrouting", {}}; Filter_chain input_chain_{"Input", {}}; Filter_chain output_chain_{"Output", {}}; + uint32_t& prerouting_dropped_; + uint32_t& postrouting_dropped_; + uint32_t& input_dropped_; + uint32_t& output_dropped_; /** All dropped packets go here */ drop_handler drop_handler_; diff --git a/src/net/ip4/ip4.cpp b/src/net/ip4/ip4.cpp index ede88f7a2a..898dbb3039 100644 --- a/src/net/ip4/ip4.cpp +++ b/src/net/ip4/ip4.cpp @@ -39,7 +39,11 @@ namespace net { packets_rx_ {Statman::get().create(Stat::UINT64, inet.ifname() + ".ip4.packets_rx").get_uint64()}, packets_tx_ {Statman::get().create(Stat::UINT64, inet.ifname() + ".ip4.packets_tx").get_uint64()}, packets_dropped_ {Statman::get().create(Stat::UINT32, inet.ifname() + ".ip4.packets_dropped").get_uint32()}, - stack_ {inet} + stack_ {inet}, + prerouting_dropped_ {Statman::get().create(Stat::UINT32, inet.ifname() + ".ip4.prerouting_dropped").get_uint32()}, + postrouting_dropped_ {Statman::get().create(Stat::UINT32, inet.ifname() + ".ip4.postrouting_dropped").get_uint32()}, + input_dropped_ {Statman::get().create(Stat::UINT32, inet.ifname() + ".ip4.input_dropped").get_uint32()}, + output_dropped_ {Statman::get().create(Stat::UINT32, inet.ifname() + ".ip4.output_dropped").get_uint32()} {} @@ -141,7 +145,10 @@ namespace net { Conntrack::Entry_ptr ct = (stack_.conntrack()) ? stack_.conntrack()->in(*packet) : nullptr; auto res = prerouting_chain_(std::move(packet), stack_, ct); - if (UNLIKELY(res == Filter_verdict_type::DROP)) return; + if (UNLIKELY(res == Filter_verdict_type::DROP)) { + prerouting_dropped_++; + return; + } Ensures(res.packet != nullptr); packet = res.release(); @@ -184,7 +191,10 @@ namespace net { if(stack_.conntrack()) stack_.conntrack()->confirm(*packet); // No need to set ct again res = input_chain_(std::move(packet), stack_, ct); - if (UNLIKELY(res == Filter_verdict_type::DROP)) return; + if (UNLIKELY(res == Filter_verdict_type::DROP)) { + input_dropped_++; + return; + } Ensures(res.packet != nullptr); packet = res.release(); @@ -236,7 +246,10 @@ namespace net { Conntrack::Entry_ptr ct = (stack_.conntrack()) ? stack_.conntrack()->in(*packet) : nullptr; auto res = output_chain_(std::move(packet), stack_, ct); - if (UNLIKELY(res == Filter_verdict_type::DROP)) return; + if (UNLIKELY(res == Filter_verdict_type::DROP)) { + output_dropped_++; + return; + } Ensures(res.packet != nullptr); packet = res.release(); @@ -272,7 +285,10 @@ namespace net { conntrack->confirm(ct->first, ct->proto) : conntrack->confirm(*packet); } auto res = postrouting_chain_(std::move(packet), stack_, ct); - if (UNLIKELY(res == Filter_verdict_type::DROP)) return; + if (UNLIKELY(res == Filter_verdict_type::DROP)) { + postrouting_dropped_++; + return; + } Ensures(res.packet != nullptr); packet = res.release(); From 2508a90c1f11268d52996be8ec6c275e65ccf2f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 18 Sep 2018 15:28:13 +0200 Subject: [PATCH 637/723] test: Add obscure test for TCP conntrack module --- test/net/integration/gateway/service.cpp | 97 +++++++++++++++++++++--- 1 file changed, 86 insertions(+), 11 deletions(-) diff --git a/test/net/integration/gateway/service.cpp b/test/net/integration/gateway/service.cpp index a63377e048..66d13f76a0 100644 --- a/test/net/integration/gateway/service.cpp +++ b/test/net/integration/gateway/service.cpp @@ -16,6 +16,7 @@ // limitations under the License. #include +#include #include using namespace net; @@ -24,22 +25,17 @@ int pongs = 0; void verify() { static int i = 0; - if (++i == 8) printf("SUCCESS\n"); + if (++i == 9) printf("SUCCESS\n"); } -#include +inline void test_tcp_conntrack(); void Service::start() { - static auto& eth0 = Inet::stack<0>(); - - static auto& eth1 = Inet::stack<1>(); - - static auto& host1 = Inet::stack<2>(); - - static auto& host2 = Inet::stack<3>(); - - eth0.conntrack()->tcp_in = net::tcp::tcp4_conntrack; + static auto& eth0 = Super_stack::get(0); + static auto& eth1 = Super_stack::get(1); + static auto& host1 = Super_stack::get(2); + static auto& host2 = Super_stack::get(3); INFO("Ping", "host1 => host2 (%s)", host2.ip_addr().to_string().c_str()); host1.icmp().ping(host2.ip_addr(), [](auto reply) { @@ -101,4 +97,83 @@ void Service::start() std::string udp_data{"yolo"}; eth1.udp().bind().sendto(eth0.ip_addr(), 3333, udp_data.data(), udp_data.size()); + + // some breathing room + Timers::oneshot(std::chrono::seconds(2), [](auto) { + test_tcp_conntrack(); + }); +} + +#include +#include + +void test_tcp_conntrack() +{ + static std::vector storage; + + // same rules still apply + static auto& eth0 = Super_stack::get(0); + static auto& eth1 = Super_stack::get(1); + static auto& host1 = Super_stack::get(2); + static auto& host2 = Super_stack::get(3); + + // retreive the shared conntrack instance + eth0.conntrack()->tcp_in = net::tcp::tcp4_conntrack; + + // eth0 won't have seen the handshake so it will drop them packets + static auto& stat = Statman::get().get_by_name("eth0.ip4.prerouting_dropped"); + static const auto drop_before = stat.get_uint32(); + + const uint16_t port {6666}; + // hack to reduce close time so we don't have to wait forever + // for the connection to close down, allowing us to end the + // test when conn is closed + host2.tcp().set_MSL(std::chrono::seconds(1)); + host2.tcp().listen(port, [](auto conn) + { + conn->on_read(1024, [conn](auto buf) { + std::string str{(const char*)buf->data(), buf->size()}; + printf("Recv %s\n", str.c_str()); + if(str == "10") + conn->close(); + }); + conn->on_close([]{ + const auto drop_after = stat.get_uint32(); + CHECKSERT((drop_after - drop_before) > 5, + "At least 6 packets has been denied(dropped) before=%u after=%u", drop_before, drop_after); + verify(); + }); + }); + + host1.tcp().connect({host2.ip_addr(), port}, [](auto conn) + { + static int i = 0; + + while(i++ < 10) + { + auto n = i; + Timers::oneshot(std::chrono::milliseconds(n*500), [n, conn](auto) + { + conn->write(std::to_string(n)); + + if(n == 5) + { + printf("Store CT state\n"); + eth0.conntrack()->serialize_to(storage); + printf("Assigning new CT, breaking established connections.\n"); + auto new_ct = std::make_shared(); + new_ct->tcp_in = net::tcp::tcp4_conntrack; + eth0.conntrack() = new_ct; + eth1.conntrack() = new_ct; + + // restore it after 3 seconds, allowing connections to get through again + Timers::oneshot(std::chrono::seconds(3), [](auto) + { + printf("Restoring CT state\n"); + eth0.conntrack()->deserialize_from(storage.data()); + }); + } + }); + } + }); } From 2d0ad38de52f1d7d0e9f9ec116457493c720cf99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 18 Sep 2018 15:39:55 +0200 Subject: [PATCH 638/723] test: Fixed typo in coment --- test/net/integration/gateway/service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net/integration/gateway/service.cpp b/test/net/integration/gateway/service.cpp index 66d13f76a0..7a3d0c4da6 100644 --- a/test/net/integration/gateway/service.cpp +++ b/test/net/integration/gateway/service.cpp @@ -117,7 +117,7 @@ void test_tcp_conntrack() static auto& host1 = Super_stack::get(2); static auto& host2 = Super_stack::get(3); - // retreive the shared conntrack instance + // retrieve the shared conntrack instance eth0.conntrack()->tcp_in = net::tcp::tcp4_conntrack; // eth0 won't have seen the handshake so it will drop them packets From 6514a8d20270ef6e4706a39f8c53504f2cf30731 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 18 Sep 2018 17:13:27 +0200 Subject: [PATCH 639/723] vmxnet3: Re-instate second RX descriptor array --- src/drivers/vmxnet3.cpp | 14 ++++++++------ src/drivers/vmxnet3.hpp | 4 ++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index e4c7666140..caf0089c82 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -51,7 +51,8 @@ struct vmxnet3_dma { struct vmxnet3_tx_comp tx_comp[VMXNET3_NUM_TX_COMP]; /** RX ring */ struct vmxnet3_rx { - struct vmxnet3_rx_desc desc[vmxnet3::NUM_RX_DESC]; + struct vmxnet3_rx_desc desc0[vmxnet3::NUM_RX_DESC]; + struct vmxnet3_rx_desc desc1[vmxnet3::NUM_RX_DESC]; struct vmxnet3_rx_comp comp[VMXNET3_NUM_RX_COMP]; }; struct vmxnet3_rx rx[vmxnet3::NUM_RX_QUEUES]; @@ -209,8 +210,8 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : for (int q = 0; q < NUM_RX_QUEUES; q++) { memset(rx[q].buffers, 0, sizeof(rx[q].buffers)); - rx[q].desc0 = &dma->rx[q].desc[0]; - rx[q].desc1 = nullptr; + rx[q].desc0 = &dma->rx[q].desc0[0]; + rx[q].desc1 = &dma->rx[q].desc1[0]; rx[q].comp = &dma->rx[q].comp[0]; rx[q].index = q; @@ -219,7 +220,7 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : queue.cfg.desc_address[1] = (uintptr_t) rx[q].desc1; queue.cfg.comp_address = (uintptr_t) rx[q].comp; queue.cfg.num_desc[0] = vmxnet3::NUM_RX_DESC; - queue.cfg.num_desc[1] = 0; + queue.cfg.num_desc[1] = vmxnet3::NUM_RX_DESC; queue.cfg.num_comp = VMXNET3_NUM_RX_COMP; queue.cfg.driver_data_len = sizeof(vmxnet3_rx_desc) + 2 * sizeof(vmxnet3_rx_desc); @@ -318,8 +319,9 @@ void vmxnet3::retrieve_hwaddr() } mac; mac.lo = mmio_read32(this->iobase + VMXNET3_VD_MAC_LO); mac.hi = mmio_read32(this->iobase + VMXNET3_VD_MAC_HI); - // ETH_ALEN = 6 - memcpy(&this->hw_addr, &mac, sizeof(hw_addr)); + // avoid memcpy() when we can just use bitwise-operators + this->hw_addr.minor = mac.lo & 0xFFFF; + this->hw_addr.major = (mac.lo >> 16) | (mac.hi << 16); INFO2("MAC address: %s", hw_addr.to_string().c_str()); } void vmxnet3::set_hwaddr(MAC::Addr& addr) diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index 2296452d72..9629ebe655 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -31,8 +31,8 @@ class vmxnet3 : public net::Link_layer using Link_protocol = Link::Protocol; static const int DRIVER_OFFSET = 2; static const int NUM_RX_QUEUES = 1; - static const int NUM_TX_DESC = 512; - static const int NUM_RX_DESC = 512; + static const int NUM_TX_DESC = 128; + static const int NUM_RX_DESC = 128; static std::unique_ptr new_instance(hw::PCI_Device& d, const uint16_t MTU) { return std::make_unique(d, MTU); } From a48c46f5209958f81082422db04e61e7523a081a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 19 Sep 2018 10:20:38 +0200 Subject: [PATCH 640/723] vmxnet3: Merge fixed RX queues into one --- src/drivers/vmxnet3.cpp | 7 +++---- src/drivers/vmxnet3.hpp | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index caf0089c82..c8538bf442 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -51,8 +51,7 @@ struct vmxnet3_dma { struct vmxnet3_tx_comp tx_comp[VMXNET3_NUM_TX_COMP]; /** RX ring */ struct vmxnet3_rx { - struct vmxnet3_rx_desc desc0[vmxnet3::NUM_RX_DESC]; - struct vmxnet3_rx_desc desc1[vmxnet3::NUM_RX_DESC]; + struct vmxnet3_rx_desc desc[vmxnet3::NUM_RX_DESC]; struct vmxnet3_rx_comp comp[VMXNET3_NUM_RX_COMP]; }; struct vmxnet3_rx rx[vmxnet3::NUM_RX_QUEUES]; @@ -210,8 +209,8 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : for (int q = 0; q < NUM_RX_QUEUES; q++) { memset(rx[q].buffers, 0, sizeof(rx[q].buffers)); - rx[q].desc0 = &dma->rx[q].desc0[0]; - rx[q].desc1 = &dma->rx[q].desc1[0]; + rx[q].desc0 = &dma->rx[q].desc[0]; + rx[q].desc1 = &dma->rx[q].desc[0]; rx[q].comp = &dma->rx[q].comp[0]; rx[q].index = q; diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index 9629ebe655..0b02f0722a 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -32,7 +32,7 @@ class vmxnet3 : public net::Link_layer static const int DRIVER_OFFSET = 2; static const int NUM_RX_QUEUES = 1; static const int NUM_TX_DESC = 128; - static const int NUM_RX_DESC = 128; + static const int NUM_RX_DESC = 512; static std::unique_ptr new_instance(hw::PCI_Device& d, const uint16_t MTU) { return std::make_unique(d, MTU); } From e0930d0aa066543b52827e7aaae1e0caba6d25dc Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 19 Sep 2018 10:54:36 +0200 Subject: [PATCH 641/723] vmxnet3: Don't refill whole TX ring --- src/drivers/vmxnet3.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index c8538bf442..7290e84c98 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -36,7 +36,7 @@ static std::vector deferred_devs; #define VMXNET3_NUM_TX_COMP vmxnet3::NUM_TX_DESC #define VMXNET3_NUM_RX_COMP vmxnet3::NUM_RX_DESC -static const int VMXNET3_TX_FILL = vmxnet3::NUM_TX_DESC; +static const int VMXNET3_TX_FILL = vmxnet3::NUM_TX_DESC-1; static const int VMXNET3_RX_FILL = vmxnet3::NUM_RX_DESC; /** From 62300c5d0537e066eb7d08822c3584e275639237 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 19 Sep 2018 16:32:19 +0200 Subject: [PATCH 642/723] linux: Add support for PGO with GCC, with script --- CMakeLists.txt | 1 + cmake/linux.service.cmake | 15 ++++++++++++- etc/linux/lxp-pgo | 4 ++++ etc/linux/lxp-run | 45 +++++++++++++++++++++++++++++++++++---- linux/CMakeLists.txt | 10 +++++++++ 5 files changed, 70 insertions(+), 5 deletions(-) create mode 100755 etc/linux/lxp-pgo diff --git a/CMakeLists.txt b/CMakeLists.txt index 12cc0156d9..0384a53774 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -275,6 +275,7 @@ install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-run DESTINATION bin) install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-callgraph DESTINATION bin) install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-debug DESTINATION bin) install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-gprof DESTINATION bin) +install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/etc/linux/lxp-pgo DESTINATION bin) # Install scripts install(PROGRAMS diff --git a/cmake/linux.service.cmake b/cmake/linux.service.cmake index 63d1ed0e04..7a0d24bf29 100644 --- a/cmake/linux.service.cmake +++ b/cmake/linux.service.cmake @@ -8,7 +8,10 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 ${COMMON}") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${COMMON}") option(DEBUGGING "Enable debugging" OFF) +option(PERFORMANCE "Enable performance mode" OFF) option(GPROF "Enable profiling with gprof" OFF) +option(PGO_ENABLE "Enable guided profiling (PGO)" OFF) +option(PGO_GENERATE "PGO is in profile generating mode" ON) option(SANITIZE "Enable undefined- and address sanitizers" OFF) option(ENABLE_LTO "Enable LTO for use with Clang/GCC" OFF) option(CUSTOM_BOTAN "Enable building with a local Botan" OFF) @@ -18,6 +21,8 @@ option(USE_LLD "Allow linking against LTO archives" OFF) if(DEBUGGING) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0") +elseif(PERFORMANCE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Ofast") endif() if (ENABLE_LTO) @@ -34,6 +39,14 @@ if(GPROF) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pg -fno-omit-frame-pointer") endif() +if (PGO_ENABLE) + if (PGO_GENERATE) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-dir=$ENV{HOME}/pgo -fprofile-generate") + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-dir=$ENV{HOME}/pgo -fprofile-use") + endif() +endif() + if(SANITIZE) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=undefined -fsanitize=address") endif() @@ -115,7 +128,7 @@ endif() if (STATIC_BUILD) set(CMAKE_SHARED_LIBRARY_LINK_C_FLAGS "") set(CMAKE_SHARED_LIBRARY_LINK_CXX_FLAGS "") - target_link_libraries(service -static-libstdc++ -static-libgcc -static) + target_link_libraries(service -static-libstdc++ -static-libgcc) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -static -pthread") set(BUILD_SHARED_LIBRARIES OFF) endif() diff --git a/etc/linux/lxp-pgo b/etc/linux/lxp-pgo new file mode 100755 index 0000000000..52c1ae777f --- /dev/null +++ b/etc/linux/lxp-pgo @@ -0,0 +1,4 @@ +#! /bin/bash +echo -e "\n>>> Running service with profile-guided optimizations" +source lxp-run pgo-gen +source lxp-run pgo-use diff --git a/etc/linux/lxp-run b/etc/linux/lxp-run index ee1e80f05c..601fea3b9f 100755 --- a/etc/linux/lxp-run +++ b/etc/linux/lxp-run @@ -1,30 +1,67 @@ #!/bin/bash set -e export GPROF=${GPROF:-"OFF"} +export PGO_EN=${PGO_EN:-"OFF"} +export PGO_GEN=${PGO_GEN:-"OFF"} export DBG=${DBG:-"OFF"} export NUM_JOBS=${NUM_JOBS:--j8} +scriptName="${0##*/}" -function make_linux(){ +function make_linux() { pushd $INCLUDEOS_SRC/linux/build - cmake -DGPROF=$GPROF -DDEBUGGING=$DBG -DCMAKE_INSTALL_PREFIX=$INCLUDEOS_PREFIX .. + cmake -DGPROF=$GPROF -DPGO_ENABLE=$PGO_EN -DPGO_GENERATE=$PGO_GEN -DDEBUGGING=$DBG -DCMAKE_INSTALL_PREFIX=$INCLUDEOS_PREFIX .. make $NUM_JOBS install popd } -function make_service(){ +function make_service() { mkdir -p build pushd build - cmake -DGPROF=$GPROF -DDEBUGGING=$DBG .. + cmake -DGPROF=$GPROF -DPGO_ENABLE=$PGO_EN -DPGO_GENERATE=$PGO_GEN -DDEBUGGING=$DBG .. make $NUM_JOBS popd } +function print_usage() { + cat < Date: Mon, 24 Sep 2018 12:09:47 +0200 Subject: [PATCH 643/723] net: BufferStore now uses internal pools instead of a chain --- api/net/buffer_store.hpp | 73 ++++------ src/drivers/e1000.cpp | 7 +- src/drivers/solo5net.cpp | 12 +- src/drivers/virtionet.cpp | 11 +- src/drivers/vmxnet3.cpp | 7 +- src/net/buffer_store.cpp | 161 +++++----------------- test/lest_util/nic_mock.hpp | 5 +- test/lest_util/packet_factory.hpp | 5 +- test/net/integration/bufstore/service.cpp | 6 +- test/net/unit/bufstore.cpp | 4 +- test/net/unit/packets.cpp | 10 +- 11 files changed, 91 insertions(+), 210 deletions(-) diff --git a/api/net/buffer_store.hpp b/api/net/buffer_store.hpp index 50364a7d27..123f745a07 100644 --- a/api/net/buffer_store.hpp +++ b/api/net/buffer_store.hpp @@ -36,65 +36,52 @@ namespace net **/ class BufferStore { public: - struct buffer_t - { - BufferStore* bufstore; - uint8_t* addr; - }; - - BufferStore(size_t num, size_t bufsize); + BufferStore(uint32_t num, uint32_t bufsize); ~BufferStore(); - buffer_t get_buffer(); + uint8_t* get_buffer(); inline void release(void*); - size_t local_buffers() const noexcept - { return poolsize_ / bufsize_; } - /** Get size of a buffer **/ - size_t bufsize() const noexcept + uint32_t bufsize() const noexcept { return bufsize_; } - size_t poolsize() const noexcept + uint32_t poolsize() const noexcept { return poolsize_; } - /** Check if a buffer belongs here */ - bool is_from_this_pool(uint8_t* addr) const noexcept { - return (addr >= this->pool_begin() - and addr < this->pool_end()); + /** Check if an address belongs to this buffer store */ + bool is_valid(uint8_t* addr) const noexcept + { + for (const auto* pool : pools_) + if ((addr - pool) % bufsize_ == 0 + && addr >= pool && addr < pool + poolsize_) return true; + return false; } - /** Check if an address is the start of a buffer */ - bool is_buffer(uint8_t* addr) const noexcept - { return (addr - pool_) % bufsize_ == 0; } - - size_t available() const noexcept; + size_t available() const noexcept + { + return this->available_.size(); + } - size_t total_buffers() const noexcept; + size_t total_buffers() const noexcept + { + return this->pool_buffers() * this->pools_.size(); + } /** move this bufferstore to the current CPU **/ void move_to_this_cpu() noexcept; private: - uint8_t* pool_begin() const noexcept { - return pool_; - } - uint8_t* pool_end() const noexcept { - return pool_begin() + poolsize_; - } - - BufferStore* get_next_bufstore(); - inline buffer_t get_buffer_directly() noexcept; - inline void release_directly(uint8_t*); - void release_internal(void*); + uint32_t pool_buffers() const noexcept { return poolsize_ / bufsize_; } + void create_new_pool(); + bool growth_enabled() const; - size_t poolsize_; - size_t bufsize_; - uint8_t* pool_; + uint32_t poolsize_; + uint32_t bufsize_; + int index = -1; std::vector available_; - std::unique_ptr next_; - int index; + std::vector pools_; #ifdef INCLUDEOS_SMP_ENABLE // has strict alignment reqs, so put at end spinlock_t plock = 0; @@ -108,16 +95,14 @@ namespace net inline void BufferStore::release(void* addr) { auto* buff = (uint8_t*) addr; - // try to release directly into pool - if (LIKELY(is_from_this_pool(buff))) { + if (LIKELY(this->is_valid(buff))) { #ifdef INCLUDEOS_SMP_ENABLE scoped_spinlock spinlock(this->plock); #endif - available_.push_back(buff); + this->available_.push_back(buff); return; } - // release via chained stores - release_internal(addr); + throw std::runtime_error("Buffer did not belong"); } } //< net diff --git a/src/drivers/e1000.cpp b/src/drivers/e1000.cpp index deef84b0c0..b5d6d64c01 100644 --- a/src/drivers/e1000.cpp +++ b/src/drivers/e1000.cpp @@ -462,18 +462,17 @@ e1000::recv_packet(uint8_t* data, uint16_t size) net::Packet_ptr e1000::create_packet(int link_offset) { - auto buffer = bufstore().get_buffer(); - auto* ptr = (net::Packet*) buffer.addr; + auto* ptr = (net::Packet*) bufstore().get_buffer(); new (ptr) net::Packet( DRIVER_OFFSET + link_offset, 0, DRIVER_OFFSET + frame_offset_link() + MTU(), - buffer.bufstore); + &bufstore()); return net::Packet_ptr(ptr); } uintptr_t e1000::new_rx_packet() { - auto* pkt = bufstore().get_buffer().addr; + auto* pkt = bufstore().get_buffer(); return (uintptr_t) &pkt[sizeof(net::Packet) + DRIVER_OFFSET]; } diff --git a/src/drivers/solo5net.cpp b/src/drivers/solo5net.cpp index 68dc032356..5ff4d16923 100644 --- a/src/drivers/solo5net.cpp +++ b/src/drivers/solo5net.cpp @@ -66,17 +66,15 @@ void Solo5Net::transmit(net::Packet_ptr pckt) net::Packet_ptr Solo5Net::create_packet(int link_offset) { - auto buffer = bufstore().get_buffer(); - auto* pckt = (net::Packet*) buffer.addr; + auto* pckt = (net::Packet*) bufstore().get_buffer(); - new (pckt) net::Packet(link_offset, 0, packet_len(), buffer.bufstore); + new (pckt) net::Packet(link_offset, 0, packet_len(), &bufstore()); return net::Packet_ptr(pckt); } net::Packet_ptr Solo5Net::recv_packet() { - auto buffer = bufstore().get_buffer(); - auto* pckt = (net::Packet*) buffer.addr; - new (pckt) net::Packet(0, MTU(), packet_len(), buffer.bufstore); + auto* pckt = (net::Packet*) bufstore().get_buffer(); + new (pckt) net::Packet(0, MTU(), packet_len(), &bufstore()); // Populate the packet buffer with new packet, if any size_t size = packet_len(); if (solo5_net_read(pckt->buf(), size, &size) == 0) { @@ -86,7 +84,7 @@ net::Packet_ptr Solo5Net::recv_packet() return net::Packet_ptr(pckt); } } - bufstore().release(buffer.addr); + bufstore().release(&pckt); return nullptr; } diff --git a/src/drivers/virtionet.cpp b/src/drivers/virtionet.cpp index 99a1b54680..5f8d22a329 100644 --- a/src/drivers/virtionet.cpp +++ b/src/drivers/virtionet.cpp @@ -142,9 +142,7 @@ VirtioNet::VirtioNet(hw::PCI_Device& d, const uint16_t /*mtu*/) rx_q.size() / 2, (uint32_t) bufstore().bufsize()); for (int i = 0; i < rx_q.size() / 2; i++) { - auto buf = bufstore().get_buffer(); - assert(bufstore().is_from_this_pool(buf.addr)); - add_receive_buffer(buf.addr); + add_receive_buffer(bufstore().get_buffer()); } // Step 4 - If there are many queues, we should negotiate the number. @@ -229,7 +227,7 @@ void VirtioNet::msix_recv_handler() Link::receive( recv_packet(res.data(), res.size()) ); // Requeue a new buffer - add_receive_buffer(bufstore().get_buffer().addr); + add_receive_buffer(bufstore().get_buffer()); // Stat increase packets received packets_rx_++; @@ -308,14 +306,13 @@ VirtioNet::recv_packet(uint8_t* data, uint16_t size) net::Packet_ptr VirtioNet::create_packet(int link_offset) { - auto buffer = bufstore().get_buffer(); - auto* ptr = (net::Packet*) buffer.addr; + auto* ptr = (net::Packet*) bufstore().get_buffer(); new (ptr) net::Packet( sizeof(virtio_net_hdr) + link_offset, 0, sizeof(virtio_net_hdr) + frame_offset_link() + MTU(), - buffer.bufstore); + &bufstore()); return net::Packet_ptr(ptr); } diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index e4c7666140..bde3186a20 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -362,7 +362,7 @@ void vmxnet3::refill(rxring_state& rxq) (rxq.producers & vmxnet3::NUM_RX_DESC) ? 0 : VMXNET3_RXF_GEN; // get a pointer to packet data - auto* pkt_data = bufstore().get_buffer().addr; + auto* pkt_data = bufstore().get_buffer(); rxq.buffers[i] = &pkt_data[sizeof(net::Packet) + DRIVER_OFFSET]; // assign rx descriptor @@ -393,13 +393,12 @@ vmxnet3::recv_packet(uint8_t* data, uint16_t size) net::Packet_ptr vmxnet3::create_packet(int link_offset) { - auto buffer = bufstore().get_buffer(); - auto* ptr = (net::Packet*) buffer.addr; + auto* ptr = (net::Packet*) bufstore().get_buffer(); new (ptr) net::Packet( DRIVER_OFFSET + link_offset, 0, DRIVER_OFFSET + frame_offset_link() + MTU(), - buffer.bufstore); + &bufstore()); return net::Packet_ptr(ptr); } diff --git a/src/net/buffer_store.cpp b/src/net/buffer_store.cpp index 94a6a97bb8..a439b8fe94 100644 --- a/src/net/buffer_store.cpp +++ b/src/net/buffer_store.cpp @@ -22,48 +22,26 @@ #include #include #include -//#define DEBUG_RELEASE -//#define DEBUG_RETRIEVE //#define DEBUG_BUFSTORE -#define ENABLE_BUFFERSTORE_CHAIN - -#ifdef DEBUG_RELEASE -#define BSD_RELEASE(fmt, ...) printf(fmt, ##__VA_ARGS__); -#else -#define BSD_RELEASE(fmt, ...) /** fmt **/ -#endif - -#ifdef DEBUG_RETRIEVE -#define BSD_GET(fmt, ...) printf(fmt, ##__VA_ARGS__); -#else -#define BSD_GET(fmt, ...) /** fmt **/ -#endif #ifdef DEBUG_BUFSTORE -#define BSD_BUF(fmt, ...) printf(fmt, ##__VA_ARGS__); +#define BSD_PRINT(fmt, ...) printf(fmt, ##__VA_ARGS__); #else -#define BSD_BUF(fmt, ...) /** fmt **/ +#define BSD_PRINT(fmt, ...) /** fmt **/ #endif namespace net { - BufferStore::BufferStore(size_t num, size_t bufsize) : + BufferStore::BufferStore(uint32_t num, uint32_t bufsize) : poolsize_ {num * bufsize}, - bufsize_ {bufsize}, - next_(nullptr) + bufsize_ {bufsize} { assert(num != 0); assert(bufsize != 0); - const size_t DATA_SIZE = poolsize_; - - this->pool_ = (uint8_t*) aligned_alloc(OS::page_size(), DATA_SIZE); - assert(this->pool_); - available_.reserve(num); - for (uint8_t* b = pool_end()-bufsize; b >= pool_begin(); b -= bufsize) { - available_.push_back(b); - } - assert(available_.capacity() == num); + + this->create_new_pool(); + assert(this->available_.capacity() == num); assert(available() == num); static int bsidx = 0; @@ -71,120 +49,51 @@ namespace net { } BufferStore::~BufferStore() { - this->next_ = nullptr; - free(this->pool_); + for (auto* pool : this->pools_) + free(pool); } - size_t BufferStore::available() const noexcept - { - auto avail = this->available_.size(); -#ifdef ENABLE_BUFFERSTORE_CHAIN - auto* parent = this; - while (parent->next_ != nullptr) { - parent = parent->next_.get(); - avail += parent->available_.size(); - } -#endif - return avail; - } - size_t BufferStore::total_buffers() const noexcept - { - size_t total = this->local_buffers(); -#ifdef ENABLE_BUFFERSTORE_CHAIN - auto* parent = this; - while (parent->next_ != nullptr) { - parent = parent->next_.get(); - total += parent->local_buffers(); - } -#endif - return total; - } - - BufferStore* BufferStore::get_next_bufstore() - { -#ifdef ENABLE_BUFFERSTORE_CHAIN - BufferStore* parent = this; - while (parent->next_ != nullptr) { - parent = parent->next_.get(); - if (!parent->available_.empty()) return parent; - } - BSD_BUF(" Allocating %lu new buffers (%lu total)\n", - local_buffers(), total_buffers() + local_buffers()); - parent->next_ = std::make_unique(local_buffers(), bufsize()); - return parent->next_.get(); -#else - return nullptr; -#endif - } - - BufferStore::buffer_t BufferStore::get_buffer_directly() noexcept - { - auto addr = available_.back(); - available_.pop_back(); - return { this, addr }; - } - - BufferStore::buffer_t BufferStore::get_buffer() + uint8_t* BufferStore::get_buffer() { #ifdef INCLUDEOS_SMP_ENABLE - scoped_spinlock spinlock(plock); + scoped_spinlock spinlock(this->plock); #endif if (UNLIKELY(available_.empty())) { -#ifdef ENABLE_BUFFERSTORE_CHAIN - auto* next = get_next_bufstore(); - if (next == nullptr) - throw std::runtime_error("Unable to create new bufstore"); - - // take buffer from external bufstore - auto buffer = next->get_buffer_directly(); - BSD_GET("%d: Gave away EXTERN %p, %lu buffers remain\n", - this->index, buffer.addr, available()); - return buffer; -#else - panic(" Buffer pool empty! Not configured to increase pool size.\n"); -#endif + if (this->growth_enabled()) + this->create_new_pool(); + else + throw std::runtime_error("This BufferStore has run out of buffers"); } - - auto buffer = get_buffer_directly(); - BSD_GET("%d: Gave away %p, %lu buffers remain\n", - this->index, buffer.addr, available()); - return buffer; + + auto* addr = available_.back(); + available_.pop_back(); + BSD_PRINT("%d: Gave away %p, %zu buffers remain\n", + this->index, addr, available()); + return addr; } - void BufferStore::release_internal(void* addr) + void BufferStore::create_new_pool() { - auto* buff = (uint8_t*) addr; - BSD_RELEASE("%d: Release %p -> ", this->index, buff); + auto* pool = (uint8_t*) aligned_alloc(OS::page_size(), poolsize_); + assert(pool != nullptr); + this->pools_.push_back(pool); -#ifdef INCLUDEOS_SMP_ENABLE - scoped_spinlock spinlock(plock); -#endif -#ifdef ENABLE_BUFFERSTORE_CHAIN - // try to release buffer on linked bufferstore - BufferStore* ptr = next_.get(); - while (ptr != nullptr) { - if (ptr->is_from_this_pool(buff)) { - BSD_RELEASE("released on other bufferstore\n"); - ptr->release_directly(buff); - return; - } - ptr = ptr->next_.get(); + for (uint8_t* b = pool; b < pool + poolsize_; b += bufsize_) { + this->available_.push_back(b); } -#endif - throw std::runtime_error("Packet did not belong"); - } - - void BufferStore::release_directly(uint8_t* buffer) - { - BSD_GET("%d: Released EXTERN %p, %lu buffers remain\n", - this->index, buffer, available()); - available_.push_back(buffer); + BSD_PRINT("%d: Creating new pool, now %zu total buffers\n", + this->index, this->total_buffers()); } void BufferStore::move_to_this_cpu() noexcept { // TODO: hmm } + + __attribute__((weak)) + bool BufferStore::growth_enabled() const { + return true; + } -} //< namespace net +} //< net diff --git a/test/lest_util/nic_mock.hpp b/test/lest_util/nic_mock.hpp index 5f32f018d4..3ca204c681 100644 --- a/test/lest_util/nic_mock.hpp +++ b/test/lest_util/nic_mock.hpp @@ -80,13 +80,12 @@ class Nic_mock : public hw::Nic { net::Packet_ptr create_packet(int) override { - auto buffer = bufstore_.get_buffer(); - auto* ptr = (net::Packet*) buffer.addr; + auto* ptr = (net::Packet*) bufstore_.get_buffer(); new (ptr) net::Packet(frame_offs_link_, 0, packet_len(), - buffer.bufstore); + &bufstore_); return net::Packet_ptr(ptr); } diff --git a/test/lest_util/packet_factory.hpp b/test/lest_util/packet_factory.hpp index 7fcf25635b..7a670bfbce 100644 --- a/test/lest_util/packet_factory.hpp +++ b/test/lest_util/packet_factory.hpp @@ -33,9 +33,8 @@ using namespace net; static net::Packet_ptr create_packet() noexcept { static net::BufferStore bufstore(BUFFER_CNT, BUFFER_SIZE); - auto buffer = bufstore.get_buffer(); - auto* ptr = (net::Packet*) buffer.addr; - new (ptr) net::Packet(PHYS_OFFSET, 0, PHYS_OFFSET + PACKET_CAPA, buffer.bufstore); + auto* ptr = (net::Packet*) bufstore.get_buffer(); + new (ptr) net::Packet(PHYS_OFFSET, 0, PHYS_OFFSET + PACKET_CAPA, &bufstore); return net::Packet_ptr(ptr); } diff --git a/test/net/integration/bufstore/service.cpp b/test/net/integration/bufstore/service.cpp index cec4bbf9d2..1db42fb59a 100644 --- a/test/net/integration/bufstore/service.cpp +++ b/test/net/integration/bufstore/service.cpp @@ -28,11 +28,9 @@ static const uint32_t BS_CHAINS = 10; static const uint32_t TOTAL_BUFFERS = BUFFER_CNT * BS_CHAINS; auto create_packet(BufferStore& bufstore) { - // get buffer (as packet + data) - auto buffer = bufstore.get_buffer(); + auto* ptr = (Packet*) bufstore.get_buffer(); // place packet at front of buffer - auto* ptr = (Packet*) buffer.addr; - new (ptr) Packet(0, 0, MTU, buffer.bufstore); + new (ptr) Packet(0, 0, MTU, &bufstore); // regular shared_ptr that calls delete on Packet return std::unique_ptr(ptr); } diff --git a/test/net/unit/bufstore.cpp b/test/net/unit/bufstore.cpp index 63f3c5a8db..cb3979a25d 100644 --- a/test/net/unit/bufstore.cpp +++ b/test/net/unit/bufstore.cpp @@ -31,7 +31,7 @@ CASE("Randomly release bufferstore buffers") for (volatile int rounds = 0; rounds < 128; rounds++) { BufferStore bufstore(BUFFER_CNT, BUFFER_SZ); - std::vector buffers; + std::vector buffers; // force out chained bufferstores for (int chain = 0; chain < BS_CHAINS; chain++) @@ -46,7 +46,7 @@ CASE("Randomly release bufferstore buffers") // release them randomly while (!buffers.empty()) { int idx = rand() % buffers.size(); - bufstore.release(buffers[idx].addr); + bufstore.release(buffers[idx]); buffers.erase(buffers.begin() + idx, buffers.begin() + idx + 1); } EXPECT(bufstore.available() == BUFFER_CNT * BS_CHAINS); diff --git a/test/net/unit/packets.cpp b/test/net/unit/packets.cpp index 879925f858..3eb2b17ac2 100644 --- a/test/net/unit/packets.cpp +++ b/test/net/unit/packets.cpp @@ -30,9 +30,8 @@ void panic(const char*) { throw std::runtime_error("panic()"); } CASE("Create empty packet") { - auto buffer = bufstore.get_buffer(); - auto* ptr = (net::Packet*) buffer.addr; - new (ptr) net::Packet(0, 0, 0, buffer.bufstore); + auto* ptr = (net::Packet*) bufstore.get_buffer(); + new (ptr) net::Packet(0, 0, 0, &bufstore); net::Packet_ptr packet(ptr); // everything is zero @@ -49,9 +48,8 @@ CASE("Create empty packet") static Packet_ptr create_packet() noexcept { - auto buffer = bufstore.get_buffer(); - auto* ptr = (net::Packet*) buffer.addr; - new (ptr) net::Packet(DRIVER_OFFSET, 0, DRIVER_OFFSET + PACKET_CAPA, buffer.bufstore); + auto* ptr = (net::Packet*) bufstore.get_buffer(); + new (ptr) net::Packet(DRIVER_OFFSET, 0, DRIVER_OFFSET + PACKET_CAPA, &bufstore); return net::Packet_ptr(ptr); } From 27f58d494761eaca679d0e27e664332294e70294 Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Tue, 25 Sep 2018 08:49:33 +0200 Subject: [PATCH 644/723] Updating NaCl (new vlan implementation which includes the possibility to route between vlans) --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index 986837d21c..8b9a7e8105 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit 986837d21ce2da280f18f5d2e52360025ed230f2 +Subproject commit 8b9a7e8105f33ac7bc676dd25bccc1396369b3c3 From cc380407c1568de9233d751b8c7d38bd61faae07 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 25 Sep 2018 10:24:01 +0200 Subject: [PATCH 645/723] nic: Add buffer_limit and sendq_limit --- api/hw/nic.hpp | 21 +++++++++++++++++++++ api/net/buffer_store.hpp | 10 ++++++---- src/drivers/vmxnet3.cpp | 17 ++++++++++++++++- src/drivers/vmxnet3.hpp | 2 ++ 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/api/hw/nic.hpp b/api/hw/nic.hpp index 6d34cb603d..23f60f8ba4 100644 --- a/api/hw/nic.hpp +++ b/api/hw/nic.hpp @@ -101,6 +101,18 @@ namespace hw { /** Overridable MTU detection function per-network **/ static uint16_t MTU_detection_override(int idx, uint16_t default_MTU); + /** Set new buffer limit, where 0 means infinite **/ + void set_buffer_limit(uint32_t new_limit) { + this->m_buffer_limit = new_limit; + } + uint32_t buffer_limit() const noexcept { return m_buffer_limit; } + + /** Set new sendq limit, where 0 means infinite **/ + void set_sendq_limit(uint32_t new_limit) { + this->m_sendq_limit = new_limit; + } + uint32_t sendq_limit() const noexcept { return m_sendq_limit; } + protected: /** * Constructor @@ -133,8 +145,17 @@ namespace hw { } } + bool buffers_still_available(uint32_t size) const noexcept { + return this->buffer_limit() == 0 || size < this->buffer_limit(); + } + bool sendq_still_available(uint32_t size) const noexcept { + return this->sendq_limit() == 0 || size < this->sendq_limit(); + } + private: int N; + uint32_t m_buffer_limit = 0; + uint32_t m_sendq_limit = 0; friend class Devices; }; diff --git a/api/net/buffer_store.hpp b/api/net/buffer_store.hpp index 123f745a07..5bcbd8d838 100644 --- a/api/net/buffer_store.hpp +++ b/api/net/buffer_store.hpp @@ -59,16 +59,18 @@ namespace net return false; } - size_t available() const noexcept - { + size_t available() const noexcept { return this->available_.size(); } - size_t total_buffers() const noexcept - { + size_t total_buffers() const noexcept { return this->pool_buffers() * this->pools_.size(); } + size_t buffers_in_use() const noexcept { + return this->total_buffers() - this->available(); + } + /** move this bufferstore to the current CPU **/ void move_to_this_cpu() noexcept; diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index bde3186a20..8e59fdb9be 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -125,6 +125,8 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : m_pcidev(d), m_mtu(mtu), stat_sendq_cur{Statman::get().create(Stat::UINT32, device_name() + ".sendq_now").get_uint32()}, stat_sendq_max{Statman::get().create(Stat::UINT32, device_name() + ".sendq_max").get_uint32()}, + stat_rx_refill_dropped{Statman::get().create(Stat::UINT64, device_name() + ".rx_refill_dropped").get_uint64()}, + stat_sendq_dropped{Statman::get().create(Stat::UINT64, device_name() + ".sendq_dropped").get_uint64()}, bufstore_{1024, buffer_size_for_mtu(mtu)} { INFO("vmxnet3", "Driver initializing (rev=%#x)", d.rev_id()); @@ -357,6 +359,14 @@ void vmxnet3::refill(rxring_state& rxq) bool added_buffers = (rxq.prod_count < VMXNET3_RX_FILL); while (rxq.prod_count < VMXNET3_RX_FILL) { + // break when not allowed to refill anymore + if (rxq.prod_count > 0 /* prevent full stop? */ + && not Nic::buffers_still_available(bufstore().buffers_in_use())) + { + stat_rx_refill_dropped += VMXNET3_RX_FILL - rxq.prod_count; + break; + } + size_t i = rxq.producers % vmxnet3::NUM_RX_DESC; const uint32_t generation = (rxq.producers & vmxnet3::NUM_RX_DESC) ? 0 : VMXNET3_RXF_GEN; @@ -520,7 +530,12 @@ bool vmxnet3::receive_handler(const int Q) void vmxnet3::transmit(net::Packet_ptr pckt_ptr) { - while (pckt_ptr != nullptr) { + while (pckt_ptr != nullptr) + { + if (not Nic::sendq_still_available(this->sendq.size())) { + stat_sendq_dropped++; + continue; + } auto tail = pckt_ptr->detach_tail(); sendq.emplace_back(std::move(pckt_ptr)); pckt_ptr = std::move(tail); diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index 2296452d72..fe4fb10512 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -143,6 +143,8 @@ class vmxnet3 : public net::Link_layer // sendq as double-ended q uint32_t& stat_sendq_cur; uint32_t& stat_sendq_max; + uint64_t& stat_rx_refill_dropped; + uint64_t& stat_sendq_dropped; std::deque sendq; net::BufferStore bufstore_; }; From ac4a095320ed5c7cc8d71030a9d28786103febbb Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 25 Sep 2018 11:31:30 +0200 Subject: [PATCH 646/723] vmxnet3: Use chain_length() for sendq_dropped stat --- src/drivers/vmxnet3.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index 8e59fdb9be..54dc147596 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -533,8 +533,8 @@ void vmxnet3::transmit(net::Packet_ptr pckt_ptr) while (pckt_ptr != nullptr) { if (not Nic::sendq_still_available(this->sendq.size())) { - stat_sendq_dropped++; - continue; + stat_sendq_dropped += pckt_ptr->chain_length(); + break; } auto tail = pckt_ptr->detach_tail(); sendq.emplace_back(std::move(pckt_ptr)); From 1fc6ebfd6a88b86365a4ca606c8d925a5b722f7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 25 Sep 2018 13:40:08 +0200 Subject: [PATCH 647/723] tools: Support multiple bridges with qemu --- etc/scripts/create_bridge.sh | 21 +++++++++++++-------- vmrunner/vmrunner.py | 13 +++++++++---- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/etc/scripts/create_bridge.sh b/etc/scripts/create_bridge.sh index e8d30c5a19..deef29cf20 100755 --- a/etc/scripts/create_bridge.sh +++ b/etc/scripts/create_bridge.sh @@ -11,15 +11,20 @@ then GATEWAY=10.0.0.1 NETMASK6=64 GATEWAY6=fe80::e823:fcff:fef4:83e7 + # Håreks cool hack: + # - First two bytes is fixed to "c001" because it's cool + # - Last four is the gateway IP, 10.0.0.1 + HWADDR=c0:01:0a:00:00:01 -elif [ $# -eq 3 ] +elif [ $# -eq 4 ] then BRIDGE=$1 NETMASK=$2 GATEWAY=$3 + HWADDR=$4 else me=`basename "$0"` - echo "Usage: $me [name netmask gateway]" + echo "Usage: $me [name netmask gateway hwaddr]" exit 1 fi @@ -28,12 +33,9 @@ if [ -n "$INCLUDEOS_BRIDGE" ]; then fi echo " Creating bridge $BRIDGE, netmask $NETMASK, gateway $GATEWAY " +if [[ -n "$GATEWAY6" ]]; then echo " ipv6 netmask $NETMASK6, gateway $GATEWAY6 " - -# Håreks cool hack: -# - First two bytes is fixed to "c001" because it's cool -# - Last four is the gateway IP, 10.0.0.1 -HWADDR=c0:01:0a:00:00:01 +fi # For later use NETWORK=10.0.0.0 @@ -87,8 +89,11 @@ else echo " Configuring network bridge (requires sudo):" sudo ifconfig $BRIDGE $GATEWAY netmask $NETMASK up || exit 1 - sudo ifconfig $BRIDGE inet6 add $GATEWAY6/$NETMASK6 + if [[ -n "$GATEWAY6" ]]; then + sudo ifconfig $BRIDGE inet6 add $GATEWAY6/$NETMASK6 + fi if uname -s | grep Darwin > /dev/null 2>&1; then + echo " Setting ether to $HWADDR" sudo ifconfig $BRIDGE ether $HWADDR || exit 1 else sudo ifconfig $BRIDGE hw ether $HWADDR || exit 1 diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index 41e04730af..ed3372e9b0 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -378,9 +378,13 @@ def mod_args(self, mods): for mod in mods]) return ["-initrd", mods_list] - def net_arg(self, backend, device, if_name = "net0", mac = None, bridge = None): - qemu_ifup = INCLUDEOS_HOME + "/includeos/scripts/qemu-ifup" - qemu_ifdown = INCLUDEOS_HOME + "/includeos/scripts/qemu-ifdown" + def net_arg(self, backend, device, if_name = "net0", mac = None, bridge = None, scripts = None): + if scripts: + qemu_ifup = scripts + "qemu-ifup" + qemu_ifdown = scripts + "qemu-ifdown" + else: + qemu_ifup = INCLUDEOS_HOME + "/includeos/scripts/qemu-ifup" + qemu_ifdown = INCLUDEOS_HOME + "/includeos/scripts/qemu-ifdown" # FIXME: this needs to get removed, e.g. fetched from the schema names = {"virtio" : "virtio-net", "vmxnet" : "vmxnet3", "vmxnet3" : "vmxnet3"} @@ -523,7 +527,8 @@ def boot(self, multiboot, debug = False, kernel_args = "", image_name = None): for net in self._config["net"]: mac = net["mac"] if "mac" in net else None bridge = net["bridge"] if "bridge" in net else None - net_args += self.net_arg(net["backend"], net["device"], "net"+str(i), mac, bridge) + scripts = net["scripts"] if "scripts" in net else None + net_args += self.net_arg(net["backend"], net["device"], "net"+str(i), mac, bridge, scripts) i+=1 mem_arg = [] From 31ae2a153120d0cbd52d3b9d9de5eb0a2656361d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 25 Sep 2018 14:42:47 +0200 Subject: [PATCH 648/723] hw: Retrieve Nic from MAC addr helper function --- api/hw/devices.hpp | 26 +++++++++++++-- api/net/super_stack.hpp | 13 +------- src/net/super_stack.cpp | 72 ++++++++++++----------------------------- 3 files changed, 46 insertions(+), 65 deletions(-) diff --git a/api/hw/devices.hpp b/api/hw/devices.hpp index 9e5c060d58..ae1bfcdf6a 100644 --- a/api/hw/devices.hpp +++ b/api/hw/devices.hpp @@ -46,6 +46,12 @@ namespace hw { static Block_device& drive(const int N) { return get(N); } + // Nic helpers + inline static int nic_index(const MAC::Addr& mac); + + static int nic_index(const std::string& mac) + { return nic_index(MAC::Addr{mac.c_str()}); } + /** List all devices (decorated, as seen in boot output) */ inline static void print_devices(); @@ -112,11 +118,14 @@ namespace hw { /** Exception thrown when a device is not found (registered) */ class Device_not_found : public std::out_of_range { public: + explicit Device_not_found(const std::string& what) + : std::out_of_range{what} + {} explicit Device_not_found(const std::string& type, const int n) - : std::out_of_range( + : Device_not_found{ std::string{"Device of type "} + type + std::string{" not found at position #"} - + std::to_string(n)) + + std::to_string(n)} {} }; //< class Device_not_found @@ -131,6 +140,19 @@ namespace hw { } } + inline int Devices::nic_index(const MAC::Addr& mac) + { + auto& nics = devices(); + int i = 0; + for(auto it = nics.begin(); it != nics.end(); it++) + { + if((*it)->mac() == mac) + return i; + i++; + } + return -1; + } + template inline void Devices::print_devices(const Device_registry& devices) { diff --git a/api/net/super_stack.hpp b/api/net/super_stack.hpp index b945fd81a2..01d1833319 100644 --- a/api/net/super_stack.hpp +++ b/api/net/super_stack.hpp @@ -64,21 +64,10 @@ class Super_stack { * @return A stack */ static Inet& get(const std::string& mac); + static Inet& get(const std::string& mac, int sub); Inet& create(hw::Nic& nic, int N, int sub); - /** - * @brief Create a stack on the given Nic, - * occupying the first free index. - * - * @param nic The nic - * - * @tparam IP version - * - * @return A stack - */ - Inet& create(hw::Nic& nic); - IP4_stacks& ip4_stacks() { return ip4_stacks_; } diff --git a/src/net/super_stack.cpp b/src/net/super_stack.cpp index 0d24fdfd66..97a7cda72c 100644 --- a/src/net/super_stack.cpp +++ b/src/net/super_stack.cpp @@ -22,7 +22,6 @@ namespace net { -// Specialization for IP4 Inet& Super_stack::create(hw::Nic& nic, int N, int sub) { INFO("Network", "Creating stack for %s on %s (MTU=%u)", @@ -52,36 +51,6 @@ Inet& Super_stack::create(hw::Nic& nic, int N, int sub) return *stacks[sub]; } -// Specialization for IP4 -Inet& Super_stack::create(hw::Nic& nic) -{ - INFO("Network", "Creating stack for %s on %s (MTU=%u)", - nic.driver_name(), nic.device_name().c_str(), nic.MTU()); - - auto inet_ = [&nic]()->auto { - switch(nic.proto()) { - case hw::Nic::Proto::ETH: - return std::make_unique(nic); - default: - throw Super_stack_err{"Nic not supported"}; - } - }(); - - // Just take the first free one.. - for(auto& stacks : inet().ip4_stacks_) - { - auto& stack = stacks[0]; - if(stack == nullptr) { - stack = std::move(inet_); - return *stack; - } - } - - // we should never reach this point - throw Super_stack_err{"There wasn't a free slot to create stack on Nic"}; -} - -// Specialization for IP4 Inet& Super_stack::get(int N) { if (N < 0 || N >= (int) hw::Devices::devices().size()) @@ -98,7 +67,6 @@ Inet& Super_stack::get(int N) return inet().create(nic, N, 0); } -// Specialization for IP4 Inet& Super_stack::get(int N, int sub) { if (N < 0 || N >= (int) hw::Devices::devices().size()) @@ -118,33 +86,35 @@ Inet& Super_stack::get(int N, int sub) + std::to_string(N) + "," + std::to_string(sub) + "]"}; } -// Specialization for IP4 Inet& Super_stack::get(const std::string& mac) { MAC::Addr link_addr{mac.c_str()}; - // Look for the stack with the same NIC - for(auto& stacks : inet().ip4_stacks_) - { - auto& stack = stacks[0]; - if(stack == nullptr) - continue; - if(stack->link_addr() == link_addr) - return *stack; - } + auto index = hw::Devices::nic_index(link_addr); - // If no stack, find the NIC - auto& devs = hw::Devices::devices(); - auto it = devs.begin(); - for(; it != devs.end(); it++) { - if((*it)->mac() == link_addr) - break; - } // If no NIC, no point looking more - if(it == devs.end()) + if(index < 0) throw Stack_not_found{"No NIC found with MAC address " + mac}; + auto& stacks = inet().ip4_stacks_.at(index); + auto& stack = stacks[0]; + if(stack != nullptr) { + Expects(stack->link_addr() == link_addr); + return *stack; + } + // If not found, create - return inet().create(*(*it)); + return inet().create(hw::Devices::nic(index), index, 0); +} + +// Duplication of code to keep sanity intact +Inet& Super_stack::get(const std::string& mac, int sub) +{ + auto index = hw::Devices::nic_index(mac); + + if(index < 0) + throw Stack_not_found{"No NIC found with MAC address " + mac}; + + return get(index, sub); } Super_stack::Super_stack() From 199c3587758f6d21e2a3bde1fa9915eeb74a0c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 25 Sep 2018 15:45:39 +0200 Subject: [PATCH 649/723] net: Remove ip4_ prefix from Super stack (it's dual stack) --- api/net/super_stack.hpp | 11 ++++++----- lib/uplink/ws_uplink.cpp | 4 ++-- src/net/configure.cpp | 2 +- src/net/super_stack.cpp | 12 ++++++------ 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/api/net/super_stack.hpp b/api/net/super_stack.hpp index 01d1833319..79d57824c9 100644 --- a/api/net/super_stack.hpp +++ b/api/net/super_stack.hpp @@ -40,8 +40,9 @@ struct Stack_not_found : public Super_stack_err class Super_stack { public: - using Stacks = std::map>; - using IP4_stacks = std::vector; + // naming is hard... + using Indexed_stacks = std::map>; + using Stacks = std::vector; public: static Super_stack& inet() @@ -68,11 +69,11 @@ class Super_stack { Inet& create(hw::Nic& nic, int N, int sub); - IP4_stacks& ip4_stacks() - { return ip4_stacks_; } + Stacks& stacks() + { return stacks_; } private: - IP4_stacks ip4_stacks_; + Stacks stacks_; Super_stack(); diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index 12e5e49589..47c9a5a019 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -517,7 +517,7 @@ namespace uplink { writer.StartArray(); - auto& stacks = net::Super_stack::inet().ip4_stacks(); + auto& stacks = net::Super_stack::inet().stacks(); for(const auto& stack : stacks) { for(const auto& pair : stack) serialize_stack(writer, pair.second); @@ -627,7 +627,7 @@ namespace uplink { std::shared_ptr get_first_conntrack() { - for(auto& stacks : net::Super_stack::inet().ip4_stacks()) { + for(auto& stacks : net::Super_stack::inet().stacks()) { for(auto& stack : stacks) { if(stack.second != nullptr and stack.second->conntrack() != nullptr) diff --git a/src/net/configure.cpp b/src/net/configure.cpp index f965141b0f..8823e215e0 100644 --- a/src/net/configure.cpp +++ b/src/net/configure.cpp @@ -67,7 +67,7 @@ void configure(const rapidjson::Value& net) Expects(net.IsArray() && "Member net is not an array"); auto configs = net.GetArray(); - if(configs.Size() > Super_stack::inet().ip4_stacks().size()) + if(configs.Size() > Super_stack::inet().stacks().size()) MYINFO("! WARNING: Found more configs than there are interfaces"); // Iterate all interfaces in config for(auto& val : configs) diff --git a/src/net/super_stack.cpp b/src/net/super_stack.cpp index 97a7cda72c..a96dd86746 100644 --- a/src/net/super_stack.cpp +++ b/src/net/super_stack.cpp @@ -27,7 +27,7 @@ Inet& Super_stack::create(hw::Nic& nic, int N, int sub) INFO("Network", "Creating stack for %s on %s (MTU=%u)", nic.driver_name(), nic.device_name().c_str(), nic.MTU()); - auto& stacks = inet().ip4_stacks_.at(N); + auto& stacks = inet().stacks_.at(N); auto it = stacks.find(sub); if(it != stacks.end() and it->second != nullptr) { @@ -57,7 +57,7 @@ Inet& Super_stack::get(int N) throw Stack_not_found{"No IP4 stack found with index: " + std::to_string(N) + ". Missing device (NIC) or driver."}; - auto& stacks = inet().ip4_stacks_.at(N); + auto& stacks = inet().stacks_.at(N); if(stacks[0] != nullptr) return *stacks[0]; @@ -73,7 +73,7 @@ Inet& Super_stack::get(int N, int sub) throw Stack_not_found{"No IP4 stack found with index: " + std::to_string(N) + ". Missing device (NIC) or driver."}; - auto& stacks = inet().ip4_stacks_.at(N); + auto& stacks = inet().stacks_.at(N); auto it = stacks.find(sub); @@ -95,7 +95,7 @@ Inet& Super_stack::get(const std::string& mac) if(index < 0) throw Stack_not_found{"No NIC found with MAC address " + mac}; - auto& stacks = inet().ip4_stacks_.at(index); + auto& stacks = inet().stacks_.at(index); auto& stack = stacks[0]; if(stack != nullptr) { Expects(stack->link_addr() == link_addr); @@ -123,8 +123,8 @@ Super_stack::Super_stack() INFO("Network", "No registered network interfaces found"); for (size_t i = 0; i < hw::Devices::devices().size(); i++) { - ip4_stacks_.emplace_back(); - ip4_stacks_.back()[0] = nullptr; + stacks_.emplace_back(); + stacks_.back()[0] = nullptr; } } From 078034f8803b8b895449273f2d39fe6369d57868 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 25 Sep 2018 15:42:44 +0200 Subject: [PATCH 650/723] statman: Refactor and simplify to use std::deque --- api/util/statman.hpp | 129 ++++++-------- linux/src/os.cpp | 4 - src/platform/x86_pc/kernel_start.cpp | 1 - src/platform/x86_pc/os.cpp | 8 - src/platform/x86_solo5/os.cpp | 4 - src/util/statman.cpp | 142 +++++++--------- src/util/statman_liu.cpp | 19 +-- .../integration/LiveUpdate/test_boot.cpp | 2 +- test/kernel/integration/LiveUpdate/vm.json | 2 +- test/lest_util/os_mock.cpp | 11 -- test/util/unit/statman.cpp | 159 ++++++------------ 11 files changed, 173 insertions(+), 308 deletions(-) diff --git a/api/util/statman.hpp b/api/util/statman.hpp index a60aa1ef54..c76ea4d60e 100644 --- a/api/util/statman.hpp +++ b/api/util/statman.hpp @@ -21,8 +21,8 @@ #include #include +#include #include -#include #include struct Stats_out_of_memory : public std::out_of_range { @@ -51,32 +51,29 @@ class Stat { }; Stat(const Stat_type type, const std::string& name); + Stat(const Stat& other); + Stat& operator=(const Stat& other); ~Stat() = default; // increment stat counter void operator++(); - /// Stat_type type() const noexcept { return type_; } - /// const char* name() const noexcept { return name_; } - /// bool unused() const noexcept { return name_[0] == 0; } - /// - float& get_float(); - - /// - uint32_t& get_uint32(); - - /// - uint64_t& get_uint64(); + const float& get_float() const; + float& get_float(); + const uint32_t& get_uint32() const; + uint32_t& get_uint32(); + const uint64_t& get_uint64() const; + uint64_t& get_uint64(); std::string to_string() const; @@ -89,32 +86,20 @@ class Stat { Stat_type type_; char name_[MAX_NAME_LEN+1]; - - Stat(const Stat& other) = delete; - Stat(const Stat&& other) = delete; - Stat& operator=(const Stat& other) = delete; - Stat& operator=(Stat&& other) = delete; - }; //< class Stat class Statman { public: - using Size_type = ptrdiff_t; - // retrieve main instance of Statman static Statman& get(); // access a Stat by a given index - const Stat& operator[] (const int i) const - { - if (i < 0 || i >= cend() - cbegin()) - throw std::out_of_range("Index " + std::to_string(i) + " was out of range"); - return stats_[i]; + const Stat& operator[] (const size_t i) const { + return m_stats.at(i); } - Stat& operator[] (const int i) - { - return const_cast(static_cast(*this)[i]); + Stat& operator[] (const size_t i) { + return m_stats.at(i); } /** @@ -122,79 +107,45 @@ class Statman { **/ Stat& create(const Stat::Stat_type type, const std::string& name); // retrieve stat based on address from stats counter: &stat.get_xxx() - Stat& get(const void* addr); + Stat& get(const Stat* addr); // if you know the name of a statistic already Stat& get_by_name(const char* name); // free/delete stat based on address from stats counter void free(void* addr); - /** - * Returns the max capacity of the container - */ - Size_type capacity() const noexcept - { return end_stats_ - stats_; } - /** * Returns the number of used elements */ - Size_type size() const noexcept - { return bitmap.count_set(); } + size_t size() const noexcept { + return m_stats.size() - m_stats[0].get_uint32(); + } /** * Returns the number of bytes necessary to dump stats */ - Size_type num_bytes() const noexcept - { return (cend() - cbegin()) * sizeof(Stat); } + size_t num_bytes() const noexcept + { return sizeof(*this) + size() * sizeof(Stat); } /** * Is true if the span stats_ contains no Stat-objects */ - bool empty() const noexcept - { return size() == 0; } - - /** - * Returns true if the container is full - */ - bool full() const noexcept - { return size() == capacity(); } + bool empty() const noexcept { return m_stats.empty(); } - /** - * Returns a pointer to the first element - */ - const Stat* cbegin() const noexcept - { return stats_; } - - /** - * Returns a pointer to after the last used element - */ - const Stat* cend() const noexcept { - int idx = bitmap.last_set() + 1; // 0 .. N-1 - return &stats_[idx]; - } - - Stat* begin() noexcept { return (Stat*) cbegin(); } - Stat* end() noexcept { return (Stat*) cend(); } + auto begin() const noexcept { return m_stats.begin(); } + auto end() const noexcept { return m_stats.end(); } + auto cbegin() const noexcept { return m_stats.cbegin(); } + auto cend() const noexcept { return m_stats.cend(); } void store(uint32_t id, liu::Storage&); void restore(liu::Restore&); - void init(const uintptr_t location, const Size_type size); - - Statman() {} - Statman(const uintptr_t location, const Size_type size) - { - init(location, size); - } - ~Statman(); - + Statman(); private: - Stat* stats_; - Stat* end_stats_; - MemBitmap::word* bdata = nullptr; - MemBitmap bitmap; + std::deque m_stats; #ifdef INCLUDEOS_SMP_ENABLE spinlock_t stlock = 0; #endif + ssize_t find_free_stat() const noexcept; Statman(const Statman& other) = delete; Statman(const Statman&& other) = delete; @@ -202,4 +153,30 @@ class Statman { Statman& operator=(Statman&& other) = delete; }; //< class Statman +inline float& Stat::get_float() { + if (UNLIKELY(type_ != FLOAT)) throw Stats_exception{"Stat type is not a float"}; + return f; +} +inline uint32_t& Stat::get_uint32() { + if (UNLIKELY(type_ != UINT32)) throw Stats_exception{"Stat type is not an uint32"}; + return ui32; +} +inline uint64_t& Stat::get_uint64() { + if (UNLIKELY(type_ != UINT64)) throw Stats_exception{"Stat type is not an uint64"}; + return ui64; +} + +inline const float& Stat::get_float() const { + if (UNLIKELY(type_ != FLOAT)) throw Stats_exception{"Stat type is not a float"}; + return f; +} +inline const uint32_t& Stat::get_uint32() const { + if (UNLIKELY(type_ != UINT32)) throw Stats_exception{"Stat type is not an uint32"}; + return ui32; +} +inline const uint64_t& Stat::get_uint64() const { + if (UNLIKELY(type_ != UINT64)) throw Stats_exception{"Stat type is not an uint64"}; + return ui64; +} + #endif //< UTIL_STATMAN_HPP diff --git a/linux/src/os.cpp b/linux/src/os.cpp index 8708ff710c..34e5ea95b4 100644 --- a/linux/src/os.cpp +++ b/linux/src/os.cpp @@ -80,13 +80,9 @@ static void begin_timer(std::chrono::nanoseconds usec) } static void stop_timers() {} -#include void OS::start(const char* cmdline) { __libc_initialized = true; - // statman - static char statman_data[1 << 16]; - Statman::get().init((uintptr_t) statman_data, sizeof(statman_data)); // setup Linux timer (with signal handler) struct sigevent sev; sev.sigev_notify = SIGEV_SIGNAL; diff --git a/src/platform/x86_pc/kernel_start.cpp b/src/platform/x86_pc/kernel_start.cpp index 137bfe03ec..5bf83e83ba 100644 --- a/src/platform/x86_pc/kernel_start.cpp +++ b/src/platform/x86_pc/kernel_start.cpp @@ -76,7 +76,6 @@ static void global_ctor_test(){ int kernel_main(int, char * *, char * *) { PRATTLE(" libc initialization complete \n"); - kernel_sanity_checks(); Expects(__global_ctors_ok == 42); extern bool __libc_initialized; __libc_initialized = true; diff --git a/src/platform/x86_pc/os.cpp b/src/platform/x86_pc/os.cpp index 727a85e1ce..de3e6b1723 100644 --- a/src/platform/x86_pc/os.cpp +++ b/src/platform/x86_pc/os.cpp @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include "cmos.hpp" @@ -38,7 +37,6 @@ #endif extern "C" void* get_cpu_esp(); -extern "C" void kernel_sanity_checks(); extern uintptr_t _start; extern uintptr_t _end; extern uintptr_t _ELF_START_; @@ -136,7 +134,6 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) extern void elf_protect_symbol_areas(); elf_protect_symbol_areas(); - memmap.assign_range({0x8000, 0xffff, "Statman"}); #if defined(ARCH_x86_64) // protect the basic pagetable used by LiveUpdate and any other // systems that need to exit long/protected mode @@ -158,11 +155,6 @@ void OS::start(uint32_t boot_magic, uint32_t boot_addr) for (const auto& entry : memmap) INFO2("%s", entry.second.to_string().c_str()); - /// STATMAN /// - PROFILE("Statman"); - /// initialize on page 9, 8 pages in size - Statman::get().init(0x8000, 0x8000); - PROFILE("Platform init"); extern void __platform_init(); __platform_init(); diff --git a/src/platform/x86_solo5/os.cpp b/src/platform/x86_solo5/os.cpp index 3edf936830..3c002cc3a3 100644 --- a/src/platform/x86_solo5/os.cpp +++ b/src/platform/x86_solo5/os.cpp @@ -94,10 +94,6 @@ void OS::start(const char* cmdline) void* esp = get_cpu_esp(); MYINFO("Stack: %p", esp); - /// STATMAN /// - /// initialize on page 7, 2 pages in size - Statman::get().init(0x6000, 0x3000); - PROFILE("Memory map"); // Assign memory ranges used by the kernel auto& memmap = memory_map(); diff --git a/src/util/statman.cpp b/src/util/statman.cpp index 4cfe3b63ee..809ee24a7b 100644 --- a/src/util/statman.cpp +++ b/src/util/statman.cpp @@ -16,141 +16,127 @@ // limitations under the License. #include -#include #include #include -__attribute__((weak)) +// this is done to make sure construction only happens here +static Statman statman_instance; Statman& Statman::get() { - static Statman statman; - return statman; + return statman_instance; } -/////////////////////////////////////////////////////////////////////////////// Stat::Stat(const Stat_type type, const std::string& name) : ui64(0), type_{type} { - if(name.size() > MAX_NAME_LEN) - throw Stats_exception{"Creating stat: Name cannot be longer than " + std::to_string(MAX_NAME_LEN) + " characters"}; + if (name.size() > MAX_NAME_LEN) + throw Stats_exception("Stat name cannot be longer than " + std::to_string(MAX_NAME_LEN) + " characters"); snprintf(name_, sizeof(name_), "%s", name.c_str()); } +Stat::Stat(const Stat& other) { + this->ui64 = other.ui64; + this->type_ = other.type_; + strncpy(this->name_, other.name_, MAX_NAME_LEN); +} +Stat& Stat::operator=(const Stat& other) { + this->ui64 = other.ui64; + this->type_ = other.type_; + strncpy(this->name_, other.name_, MAX_NAME_LEN); + return *this; +} void Stat::operator++() { switch (type_) { case UINT32: ui32++; break; case UINT64: ui64++; break; case FLOAT: f += 1.0f; break; - default: throw Stats_exception{"Incrementing stat: Invalid Stat_type"}; + default: throw Stats_exception("Invalid stat type encountered when incrementing"); } } -float& Stat::get_float() { - if (type_ not_eq FLOAT) - throw Stats_exception{"Get stat: Stat_type is not a float"}; - - return f; -} - -uint32_t& Stat::get_uint32() { - if (type_ not_eq UINT32) - throw Stats_exception{"Get stat: Stat_type is not an uint32_t"}; - - return ui32; -} - -uint64_t& Stat::get_uint64() { - if (type_ not_eq UINT64) - throw Stats_exception{"Get stat: Stat_type is not an uint64_t"}; - - return ui64; -} - std::string Stat::to_string() const { switch (this->type_) { case UINT32: return std::to_string(ui32); case UINT64: return std::to_string(ui64); case FLOAT: return std::to_string(f); - default: return "Invalid Stat_type"; + default: return "Unknown stat type"; } } /////////////////////////////////////////////////////////////////////////////// - -void Statman::init(const uintptr_t start, const Size_type num_bytes) -{ - if (num_bytes < 0) - throw Stats_exception("Can't initialize statman with negative size"); - - const int N = num_bytes / sizeof(Stat); - this->stats_ = reinterpret_cast(start); - this->end_stats_ = this->stats_ + N; - - // create bitmap - int chunks = N / (sizeof(MemBitmap::word) * 8) + 1; - delete[] bdata; - bdata = new MemBitmap::word[chunks](); - bitmap = MemBitmap(bdata, chunks); - - INFO("Statman", "Initialized with %u stats capacity", (uint32_t) capacity()); -} -Statman::~Statman() -{ - delete[] bdata; +Statman::Statman() { + this->create(Stat::UINT32, "statman.unused_stats"); } -Stat& Statman::create(const Stat::Stat_type type, const std::string& name) { +Stat& Statman::create(const Stat::Stat_type type, const std::string& name) +{ #ifdef INCLUDEOS_SMP_ENABLE volatile scoped_spinlock lock(this->stlock); #endif if (name.empty()) throw Stats_exception("Cannot create Stat with no name"); - const int idx = bitmap.first_free(); - if (idx == -1 || idx >= capacity()) - throw Stats_out_of_memory{}; + const ssize_t idx = this->find_free_stat(); + if (idx < 0) { + m_stats.emplace_back(type, name); + return m_stats.back(); + } // note: we have to create this early in case it throws - auto& stat = *new (&stats_[idx]) Stat(type, name); - bitmap.set(idx); + auto& stat = *new (&m_stats[idx]) Stat(type, name); + m_stats[0].get_uint32()--; // decrease unused stats return stat; } -Stat& Statman::get(const void* addr) +Stat& Statman::get(const Stat* st) { - auto* st = (Stat*) addr; - auto mod = ((uintptr_t) st - (uintptr_t) stats_) % sizeof(Stat); - - if (st >= cbegin() && st < cend()) - if (mod == 0) // verify stat boundary - { - if (st->unused() == false) - return *st; - else - throw Stats_exception{"Accessing deleted stat"}; +#ifdef INCLUDEOS_SMP_ENABLE + volatile scoped_spinlock lock(this->stlock); +#endif + for (auto& stat : this->m_stats) { + if (&stat == st) { + if (stat.unused() == false) + return stat; + throw Stats_exception("Accessing deleted stat"); + } } - - throw std::out_of_range("Address out of range"); + throw std::out_of_range("Not a valid stat in this statman instance"); } Stat& Statman::get_by_name(const char* name) { - for (auto* st = begin(); st < end(); st++) +#ifdef INCLUDEOS_SMP_ENABLE + volatile scoped_spinlock lock(this->stlock); +#endif + for (auto& stat : this->m_stats) { - if (st->unused() == false) - if (strncmp(st->name(), name, Stat::MAX_NAME_LEN) == 0) - return *st; + if (stat.unused() == false) + if (strncmp(stat.name(), name, Stat::MAX_NAME_LEN) == 0) + return stat; } throw std::out_of_range("No stat found with exact given name"); } void Statman::free(void* addr) { - auto& stat = this->get(addr); - auto idx = &stat - cbegin(); - + auto& stat = this->get((Stat*) addr); +#ifdef INCLUDEOS_SMP_ENABLE + volatile scoped_spinlock lock(this->stlock); +#endif // delete entry new (&stat) Stat(Stat::FLOAT, ""); - bitmap.reset(idx); + m_stats[0].get_uint32()++; // increase unused stats +} + +ssize_t Statman::find_free_stat() const noexcept +{ +#ifdef INCLUDEOS_SMP_ENABLE + volatile scoped_spinlock lock(this->stlock); +#endif + for (size_t i = 0; i < this->m_stats.size(); i++) + { + if (m_stats[i].unused()) return i; + } + return -1; } diff --git a/src/util/statman_liu.cpp b/src/util/statman_liu.cpp index bac4a8ba58..d1f1f0aabc 100644 --- a/src/util/statman_liu.cpp +++ b/src/util/statman_liu.cpp @@ -3,28 +3,21 @@ void Statman::store(uint32_t id, liu::Storage& store) { - store.add_buffer(id, this->cbegin(), this->num_bytes()); + store.add_vector(id, {m_stats.begin(), m_stats.end()}); } void Statman::restore(liu::Restore& store) { - auto buffer = store.as_buffer(); store.go_next(); + auto stats = store.as_vector(); - assert(buffer.size() % sizeof(Stat) == 0); - const size_t count = buffer.size() / sizeof(Stat); - - const Stat* ptr = (Stat*) buffer.data(); - const Stat* end = ptr + count; - - for (; ptr < end; ptr++) + for (auto& merge_stat : stats) { try { - auto& stat = this->get_by_name(ptr->name()); - std::memcpy(&stat, ptr, sizeof(Stat)); + // TODO: merge here + this->get_by_name(merge_stat.name()) = merge_stat; } catch (const std::exception& e) { - auto& stat = this->create(ptr->type(), ptr->name()); - std::memcpy(&stat, ptr, sizeof(Stat)); + this->create(merge_stat.type(), merge_stat.name()) = merge_stat; } } } diff --git a/test/kernel/integration/LiveUpdate/test_boot.cpp b/test/kernel/integration/LiveUpdate/test_boot.cpp index ea974ad41e..fb0be7f597 100644 --- a/test/kernel/integration/LiveUpdate/test_boot.cpp +++ b/test/kernel/integration/LiveUpdate/test_boot.cpp @@ -59,7 +59,7 @@ static void boot_resume_all(Restore& thing) #endif // statman auto& stm = Statman::get(); - stm.restore(thing); + stm.restore(thing); thing.go_next(); auto& stat = stm.get_by_name("system.updates"); assert(stat.get_uint32() > 0); thing.pop_marker(); diff --git a/test/kernel/integration/LiveUpdate/vm.json b/test/kernel/integration/LiveUpdate/vm.json index 9b1533e099..4726a08c17 100644 --- a/test/kernel/integration/LiveUpdate/vm.json +++ b/test/kernel/integration/LiveUpdate/vm.json @@ -3,5 +3,5 @@ "net" : [ {"device" : "virtio"} ], - "mem" : 64 + "mem" : 256 } diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index 9265a7face..e8dde23e45 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -34,17 +34,6 @@ void* aligned_alloc(size_t alignment, size_t size) { char _DISK_START_; char _DISK_END_; -#include -Statman& Statman::get() { - static uintptr_t start {0}; - static const size_t memsize = 0x1000000; - if (!start) { - start = (uintptr_t) malloc(memsize); - } - static Statman statman_{start, memsize / sizeof(Stat)}; - return statman_; -} - /// RTC /// #include RTC::timestamp_t RTC::booted_at = 0; diff --git a/test/util/unit/statman.cpp b/test/util/unit/statman.cpp index 4484bfed37..c399d300f1 100644 --- a/test/util/unit/statman.cpp +++ b/test/util/unit/statman.cpp @@ -18,50 +18,19 @@ #include #include -const Statman::Size_type NUM_BYTES_GIVEN {1000}; - using namespace std; CASE( "Creating Statman objects" ) { GIVEN( "A fixed range of memory and its start position" ) { - auto* buffer = new char[NUM_BYTES_GIVEN]; - uintptr_t start = (uintptr_t) buffer; - Statman::Size_type expected_num_elements = NUM_BYTES_GIVEN / sizeof(Stat); - WHEN("Creating Statman") { - Statman statman_{start, NUM_BYTES_GIVEN}; - - EXPECT(statman_.capacity() == expected_num_elements); - EXPECT(statman_.size() == 0); - EXPECT(statman_.num_bytes() == 0); - - EXPECT(statman_.empty()); - EXPECT(!statman_.full()); - } - - WHEN( "Creating Statman that is given 0 bytes" ) - { - Statman statman_{start, 0}; - - EXPECT(statman_.capacity() == 0); - EXPECT(statman_.size() == 0); - EXPECT(statman_.num_bytes() == 0); - EXPECT(statman_.empty()); - EXPECT(statman_.full()); - - EXPECT_THROWS(Stat& stat = statman_.create(Stat::UINT32, "some.new.stat")); - EXPECT_THROWS(statman_[0]); - } - - WHEN( "Creating Statman with a negative size an exception is thrown" ) - { - EXPECT_THROWS_AS((Statman{start, -1}), Stats_exception); + Statman statman_; + EXPECT(statman_.size() == 1); + EXPECT(statman_.num_bytes() == (sizeof(Statman) + sizeof(Stat))); + EXPECT(not statman_.empty()); } - - delete[] buffer; } } @@ -69,16 +38,11 @@ CASE( "Creating and running through three Stats using Statman iterators begin an { GIVEN( "A fixed range of memory and its start position" ) { - void* buffer = malloc(NUM_BYTES_GIVEN); - uintptr_t start = (uintptr_t) buffer; - WHEN( "Creating Statman" ) { - Statman statman_{start, NUM_BYTES_GIVEN}; - - EXPECT(statman_.empty()); - EXPECT_NOT(statman_.full()); - EXPECT(statman_.size() == 0); + Statman statman_; + EXPECT(not statman_.empty()); + EXPECT(statman_.size() == 1); THEN( "A Stat can be created" ) { @@ -86,8 +50,7 @@ CASE( "Creating and running through three Stats using Statman iterators begin an EXPECT(stat.get_uint32() == 0); EXPECT_NOT(statman_.empty()); - EXPECT_NOT(statman_.full()); - EXPECT(statman_.size() == 1); + EXPECT(statman_.size() == 2); EXPECT_THROWS(stat.get_uint64()); EXPECT_THROWS(stat.get_float()); EXPECT(stat.get_uint32() == 0); @@ -110,8 +73,7 @@ CASE( "Creating and running through three Stats using Statman iterators begin an stat3.get_float()++; EXPECT_NOT(statman_.empty()); - EXPECT_NOT(statman_.full()); - EXPECT(statman_.size() == 3); + EXPECT(statman_.size() == 4); AND_THEN( "The registered Stats can be iterated through and displayed" ) { @@ -119,22 +81,22 @@ CASE( "Creating and running through three Stats using Statman iterators begin an for (auto it = statman_.begin(); it != statman_.end(); ++it) { - Stat& s = *it; + const Stat& s = *it; - if (i == 0) + if (i == 1) { EXPECT(s.name() == "net.tcp.dropped"s); EXPECT(s.get_uint32() == 2); EXPECT_THROWS(s.get_uint64()); EXPECT_THROWS(s.get_float()); } - else if (i == 1) + else if (i == 2) { EXPECT(s.name() == "net.tcp.bytes_transmitted"s); EXPECT(s.get_uint64() == 0); EXPECT_THROWS(s.get_float()); } - else if (i == 2) + else if (i == 3) { EXPECT(s.name() == "net.tcp.average"s); EXPECT(s.get_float() == 2.0f); @@ -142,33 +104,31 @@ CASE( "Creating and running through three Stats using Statman iterators begin an EXPECT_THROWS(s.get_uint64()); } else { - EXPECT(i < 3); + EXPECT(i == 0); } i++; } - EXPECT(i == 3); + EXPECT(i == 4); // note: if you move this, it might try to delete // the stats before running the above AND_THEN("Delete the stats") { + EXPECT(statman_.size() == 4); + statman_.free(&statman_[1]); EXPECT(statman_.size() == 3); - statman_.free(&statman_[0]); + statman_.free(&statman_[2]); EXPECT(statman_.size() == 2); - statman_.free(&statman_[1]); + statman_.free(&statman_[3]); EXPECT(statman_.size() == 1); - statman_.free(&statman_[2]); - EXPECT(statman_.size() == 0); } } } } } } - - free(buffer); } } @@ -176,25 +136,15 @@ CASE( "Filling Statman with Stats and running through Statman using iterators be { GIVEN( "A fixed range of memory and its start position" ) { - void* buffer = malloc(NUM_BYTES_GIVEN); - uintptr_t start = (uintptr_t) buffer; - Statman::Size_type expected_num_elements = NUM_BYTES_GIVEN / sizeof(Stat); - WHEN( "Creating Statman" ) { - Statman statman_{start, NUM_BYTES_GIVEN}; - - EXPECT(statman_.empty()); - EXPECT_NOT(statman_.full()); - EXPECT(statman_.size() == 0); + Statman statman_; + EXPECT(not statman_.empty()); AND_WHEN( "Statman is filled with Stats" ) { - EXPECT(statman_.empty()); - EXPECT(statman_.capacity() == expected_num_elements); - // fill it to the brim - for (int i = 0; i < statman_.capacity(); i++) + for (int i = 1; i < 101; i++) { EXPECT(statman_.size() == i); @@ -214,45 +164,38 @@ CASE( "Filling Statman with Stats and running through Statman using iterators be THEN("Statman is full and the Stats can be displayed using Statman iterators begin and end") { EXPECT_NOT(statman_.empty()); - EXPECT(statman_.full()); - EXPECT(statman_.size() == statman_.capacity()); + EXPECT(statman_.size() == 101); int j = 0; - for (auto it = statman_.begin(); it != statman_.end(); ++it) { - Stat& stat = *it; + const Stat& stat = *it; - EXPECT(stat.name() == "net.tcp." + std::to_string(j)); - EXPECT_NOT(stat.type() == Stat::UINT64); + if (j > 0) { + EXPECT(stat.name() == "net.tcp." + std::to_string(j)); + EXPECT_NOT(stat.type() == Stat::UINT64); - if(j % 2 == 0) - { - EXPECT(stat.type() == Stat::UINT32); - EXPECT(stat.get_uint32() == 2); - EXPECT_THROWS(stat.get_uint64()); - EXPECT_THROWS(stat.get_float()); - } - else - { - EXPECT(stat.type() == Stat::FLOAT); - EXPECT(stat.get_float() == 1.0f); - EXPECT_THROWS(stat.get_uint32()); - EXPECT_THROWS(stat.get_uint64()); + if(j % 2 == 0) + { + EXPECT(stat.type() == Stat::UINT32); + EXPECT(stat.get_uint32() == 2); + EXPECT_THROWS(stat.get_uint64()); + EXPECT_THROWS(stat.get_float()); + } + else + { + EXPECT(stat.type() == Stat::FLOAT); + EXPECT(stat.get_float() == 1.0f); + EXPECT_THROWS(stat.get_uint32()); + EXPECT_THROWS(stat.get_uint64()); + } } j++; } - - AND_WHEN( "A Stat is created when Statman is full an exception is thrown" ) - { - EXPECT_THROWS_AS(statman_.create(Stat::UINT64, "not.room"), Stats_out_of_memory); - } } } } - - free(buffer); } } @@ -260,17 +203,14 @@ CASE("A Stat is accessible through index operator") { GIVEN( "A fixed range of memory and its start position" ) { - char buffer[NUM_BYTES_GIVEN]; - uintptr_t start = (uintptr_t) buffer; - WHEN("Creating Stat with Statman") { - Statman statman_{start, NUM_BYTES_GIVEN}; + Statman statman_; Stat& stat = statman_.create(Stat::UINT32, "one.stat"); THEN("The created Stat can be accessed via the index operator") { - Stat& s = statman_[0]; + Stat& s = statman_[1]; EXPECT(s.get_uint32() == stat.get_uint32()); EXPECT(s.name() == std::string("one.stat")); @@ -281,8 +221,7 @@ CASE("A Stat is accessible through index operator") CASE("stats names can only be MAX_NAME_LEN characters long") { - char buffer[8192]; - Statman statman_((uintptr_t) buffer, sizeof(buffer)); + Statman statman_; // ok std::string statname1 {"a.stat"}; Stat& stat1 = statman_.create(Stat::UINT32, statname1); @@ -299,11 +238,9 @@ CASE("stats names can only be MAX_NAME_LEN characters long") CASE("get(addr) returns reference to stat, throws if not present") { - char buffer[8192]; - Statman statman_((uintptr_t) buffer, sizeof(buffer)); - - EXPECT_THROWS(statman_.get((void*) 0)); - EXPECT_THROWS(statman_.get(statman_.begin())); + Statman statman_; + EXPECT_THROWS(statman_.get((Stat*) 0)); + EXPECT_THROWS(statman_.get(&statman_[1])); Stat& stat1 = statman_.create(Stat::UINT32, "some.important.stat"); Stat& stat2 = statman_.create(Stat::UINT64, "other.important.stat"); Stat& stat3 = statman_.create(Stat::FLOAT, "very.important.stat"); @@ -313,7 +250,7 @@ CASE("get(addr) returns reference to stat, throws if not present") EXPECT_NO_THROW(statman_.get(&stat1)); EXPECT_NO_THROW(statman_.get(&stat2)); EXPECT_NO_THROW(statman_.get(&stat3)); - EXPECT_THROWS(statman_.get(statman_.end())); + EXPECT_THROWS(statman_.get(&*statman_.end())); // Can't create stats with empty name EXPECT_THROWS_AS(statman_.create(Stat::UINT32, ""), Stats_exception); From 0485617ee324f88a5a6d81f742cb1e13d37569ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 25 Sep 2018 15:52:26 +0200 Subject: [PATCH 651/723] test: Removed Super stack test case for creating Stack without index (no longer a feature) --- test/net/unit/super_stack.cpp | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/test/net/unit/super_stack.cpp b/test/net/unit/super_stack.cpp index 54e62d1c27..c713b6f0c9 100644 --- a/test/net/unit/super_stack.cpp +++ b/test/net/unit/super_stack.cpp @@ -36,7 +36,7 @@ CASE("Super stack functionality") nics.push_back(std::make_unique()); // 3 stacks are preallocated - EXPECT(Super_stack::inet().ip4_stacks().size() == 3); + EXPECT(Super_stack::inet().stacks().size() == 3); // Retreiving the first stack creates an interface on the first nic auto& stack1 = Super_stack::get(0); @@ -81,19 +81,5 @@ CASE("Super stack functionality") } EXPECT(stack_err == true); - // Also possible to create without assigning index, which means it takes the first free one - auto& custom_created_stack = Super_stack::inet().create(my_nic); - EXPECT(&custom_created_stack == &Super_stack::get(1)); - - // Not allowed to create if all indexes are occupied tho - Super_stack::get(2); // occupy the last free one - stack_err = false; - try { - Super_stack::inet().create(my_nic); - } catch(const Super_stack_err&) { - stack_err = true; - } - EXPECT(stack_err == true); - } From 7266d14829f832c35b4cc512bd3814eb8344a853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 25 Sep 2018 20:51:34 +0200 Subject: [PATCH 652/723] hw: Remove unecessary helper func in devices --- api/hw/devices.hpp | 3 --- src/net/super_stack.cpp | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/api/hw/devices.hpp b/api/hw/devices.hpp index ae1bfcdf6a..8fc78cd9d6 100644 --- a/api/hw/devices.hpp +++ b/api/hw/devices.hpp @@ -49,9 +49,6 @@ namespace hw { // Nic helpers inline static int nic_index(const MAC::Addr& mac); - static int nic_index(const std::string& mac) - { return nic_index(MAC::Addr{mac.c_str()}); } - /** List all devices (decorated, as seen in boot output) */ inline static void print_devices(); diff --git a/src/net/super_stack.cpp b/src/net/super_stack.cpp index a96dd86746..8e56ed3cfd 100644 --- a/src/net/super_stack.cpp +++ b/src/net/super_stack.cpp @@ -109,7 +109,7 @@ Inet& Super_stack::get(const std::string& mac) // Duplication of code to keep sanity intact Inet& Super_stack::get(const std::string& mac, int sub) { - auto index = hw::Devices::nic_index(mac); + auto index = hw::Devices::nic_index(mac.c_str()); if(index < 0) throw Stack_not_found{"No NIC found with MAC address " + mac}; From 2e3ca2ef1a6d26a3a47d53df8d1810cdfa972b4e Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 25 Sep 2018 21:51:32 +0200 Subject: [PATCH 653/723] kernel: fix wrong calculation of heap end --- api/util/alloc_buddy.hpp | 18 ++++++++++++++---- src/kernel/syscalls.cpp | 4 ++-- test/stress/service.cpp | 13 +++++++++++-- test/util/unit/buddy_alloc_test.cpp | 22 ++++++++++++++++------ 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index a0423c52c1..c0c78fb702 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -631,12 +631,22 @@ namespace os::mem::buddy { } Addr_t highest_used_r() const noexcept { - if (is_taken() or not is_parent()) + if (is_free()) { + return my_addr_; + } + + if (is_taken() or not is_parent()) { return my_addr_ + my_size_; + } + + auto right_ = right(); + if (not right_.is_free()) { + return right_.highest_used_r(); + } - auto rhs = right().highest_used_r(); - if (rhs > my_addr_ + my_size_) return rhs; - return left().highest_used_r(); + auto left_ = left(); + Expects(not left_.is_free()); + return left_.highest_used_r(); } std::string to_string() const { diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index 9c3211faa1..fb62db06e9 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -139,9 +139,9 @@ void panic(const char* why) uintptr_t heap_total = OS::heap_max() - OS::heap_begin(); fprintf(stderr, "Heap is at: %p / %p (diff=%lu)\n", (void*) OS::heap_end(), (void*) OS::heap_max(), (ulong) (OS::heap_max() - OS::heap_end())); - fprintf(stderr, "Heap usage: %lu / %lu Kb\n", // (%.2f%%)\n", + fprintf(stderr, "Heap area: %lu / %lu Kb (allocated %zu kb)\n\n", // (%.2f%%)\n", (ulong) (OS::heap_end() - OS::heap_begin()) / 1024, - (ulong) heap_total / 1024); //, total * 100.0); + (ulong) heap_total / 1024, OS::heap_usage() / 1024); //, total * 100.0); print_backtrace(); fflush(stderr); diff --git a/test/stress/service.cpp b/test/stress/service.cpp index 1d81a538c0..1e603bed97 100644 --- a/test/stress/service.cpp +++ b/test/stress/service.cpp @@ -71,8 +71,12 @@ uint64_t TCP_BYTES_SENT = 0; void print_memuse(uintptr_t u) { auto end = OS::heap_end(); - printf("Current memory usage: %s (%zi b) heap_end: 0x%zx lstack chunks: (%s)\n", - util::Byte_r(u).to_string().c_str(), u, end, util::Byte_r(end).to_string().c_str()); + auto bytes_used = OS::heap_end() - OS::heap_begin(); + auto kb_used = bytes_used / 1024; + + printf("Current memory usage: %s (%zi b) heap_end: 0x%zx (%s) calculated used: %zu (%zu kb)\n", + util::Byte_r(u).to_string().c_str(), u, end, + util::Byte_r(end).to_string().c_str(), bytes_used, kb_used); } void Service::start(const std::string&) @@ -80,6 +84,7 @@ void Service::start(const std::string&) using namespace util::literals; // Allocation / free spam to warm up auto initial_memuse = OS::heap_usage(); + auto initial_highest_used = OS::heap_end(); print_memuse(initial_memuse); std::array allocs {}; @@ -100,6 +105,10 @@ void Service::start(const std::string&) Expects(memuse > initial_memuse); } + // Verify new used heap area covers recent heap growth + Expects(OS::heap_end() - initial_highest_used >= + OS::heap_usage() - initial_memuse); + auto high_memuse = OS::heap_usage(); Expects(high_memuse >= (chunksize * allocs.size()) + initial_memuse); diff --git a/test/util/unit/buddy_alloc_test.cpp b/test/util/unit/buddy_alloc_test.cpp index 2de205742b..b80109ddd9 100644 --- a/test/util/unit/buddy_alloc_test.cpp +++ b/test/util/unit/buddy_alloc_test.cpp @@ -58,12 +58,6 @@ CASE("mem::buddy init allocator"){ EXPECT(bool(alloc.root())); - printf("Allocator \n"); - printf("Node count: %i \n", alloc.node_count()); - printf("Pool size : %i \n", alloc.pool_size()); - - //std::cout << alloc.summary() << "\n"; - EXPECT(alloc.root().height() == 1); EXPECT(not alloc.root().is_leaf()); EXPECT(not alloc.root().is_leaf()); @@ -108,6 +102,7 @@ CASE("mem::buddy basic allocation / deallocation"){ auto addr = alloc.allocate(sz); EXPECT(addr); EXPECT(alloc.in_range(addr)); + EXPECT(alloc.highest_used() == (uintptr_t)addr + sz); addresses.push_back(addr); sum += sz; EXPECT(alloc.bytes_used() == sum); @@ -183,6 +178,21 @@ CASE("mem::buddy random ordered allocation then deallocation"){ std::cout << alloc.draw_tree(); #endif + int highest_i = 0; + void* highest = nullptr; + + for (int i = 0; i < addresses.size(); i++) { + if (addresses.at(i) > highest) { + highest = addresses.at(i); + highest_i = i; + } + } + + uintptr_t hi_used = (uintptr_t)addresses.at(highest_i) + sizes.at(highest_i); + EXPECT(hi_used == alloc.highest_used()); + auto computed_use = hi_used - alloc.addr_begin(); + EXPECT(computed_use >= alloc.bytes_used()); + // Deallocate for (int i = 0; i < addresses.size(); i++) { auto addr = addresses.at(i); From 6239359dd16310fc1f26b44497d697d4b0ffeb1e Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Tue, 25 Sep 2018 22:07:19 +0200 Subject: [PATCH 654/723] test: add OS::heap_usage to os_mock --- test/lest_util/os_mock.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index 9265a7face..f34fe388cc 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -221,6 +221,11 @@ uintptr_t OS::heap_begin() noexcept { uintptr_t OS::heap_end() noexcept { return 1 << 30; } + +size_t OS::heap_usage() noexcept { + return OS::heap_end(); +} + uintptr_t OS::heap_max() noexcept { return -1; } From 05604876f494b223ef5f775af10c560fac025696 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 26 Sep 2018 10:04:39 +0200 Subject: [PATCH 655/723] test: update configure test to not use ip4_stacks() --- test/net/integration/configure/service.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/net/integration/configure/service.cpp b/test/net/integration/configure/service.cpp index 9a57288d83..757d4d6ddb 100644 --- a/test/net/integration/configure/service.cpp +++ b/test/net/integration/configure/service.cpp @@ -24,7 +24,7 @@ void Service::start() { using namespace net; - auto& stacks = Super_stack::inet().ip4_stacks(); + auto& stacks = Super_stack::inet().stacks(); CHECKSERT(stacks.size() == 6, "There are 6 interfaces"); INFO("Test", "Verify eth0"); From f657782944f2cb3cce5ef5dc6819fce3def6b590 Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Wed, 26 Sep 2018 12:08:30 +0200 Subject: [PATCH 656/723] Updating NaCl commit (limit members in Iface and vlan-Iface with mac address as index) --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index 8b9a7e8105..1e338e80e7 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit 8b9a7e8105f33ac7bc676dd25bccc1396369b3c3 +Subproject commit 1e338e80e7b8b9839da74972a58ec9da68321d50 From 2766d3d8ddb6176c6174bb723e95bff524fbc6b6 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 26 Sep 2018 14:55:26 +0200 Subject: [PATCH 657/723] elf: Add print_backtrace2() with stdout function as parameter --- src/kernel/elf.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/kernel/elf.cpp b/src/kernel/elf.cpp index c52289046e..1c9b180408 100644 --- a/src/kernel/elf.cpp +++ b/src/kernel/elf.cpp @@ -237,7 +237,7 @@ bool Elf::verify_symbols() return get_parser().verify_symbols(); } -void print_backtrace() +void print_backtrace2(void(*stdout_function)(const char*, size_t)) { char _symbol_buffer[8192]; char _btrace_buffer[8192]; @@ -255,7 +255,7 @@ void print_backtrace() int len = snprintf(_btrace_buffer, sizeof(_btrace_buffer),\ "[%d] 0x%08x + 0x%.3x: %s\n", \ N, symb.addr, symb.offset, symb.name);\ - write(1, _btrace_buffer, len); + stdout_function(_btrace_buffer, len); #elif defined(__LP64__) #define PRINT_TRACE(N, ra) \ auto symb = Elf::safe_resolve_symbol( \ @@ -263,7 +263,7 @@ void print_backtrace() int len = snprintf(_btrace_buffer, sizeof(_btrace_buffer),\ "[%d] 0x%016lx + 0x%.3x: %s\n", \ N, symb.addr, symb.offset, symb.name);\ - write(1, _btrace_buffer, len); + stdout_function(_btrace_buffer, len); #else #error "Implement me" #endif @@ -302,6 +302,12 @@ void print_backtrace() PRINT_TRACE(14, ra); }}}}}}}}}}}}}}} } +void print_backtrace() +{ + print_backtrace2([] (const char* text, size_t length) { + write(1, text, length); + }); +} void Elf::print_info() { From fe49e585ea222759d7166c1ab6c5d4a2c21c3a08 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 26 Sep 2018 15:19:18 +0200 Subject: [PATCH 658/723] liveupdate: More information when throwing exceptions --- lib/LiveUpdate/resume.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/LiveUpdate/resume.cpp b/lib/LiveUpdate/resume.cpp index 34a9bfd985..46016b6510 100644 --- a/lib/LiveUpdate/resume.cpp +++ b/lib/LiveUpdate/resume.cpp @@ -60,7 +60,7 @@ bool LiveUpdate::resume(std::string key, resume_func func) fprintf(stderr, "WARNING: LiveUpdate storage area inside heap (margin: %ld)\n", (long int) (OS::heap_end() - (uintptr_t) location)); - throw std::runtime_error("LiveUpdate storage area inside heap"); + throw std::runtime_error("LiveUpdate::resume(): Storage area inside heap"); } return resume_helper(location, std::move(key), func); } @@ -123,13 +123,13 @@ int Restore::as_int() const // this type uses the length field directly as value to save space if (ent->type == TYPE_INTEGER) return ent->len; - throw std::runtime_error("LiveUpdate: Incorrect type " + std::to_string(ent->type)); + throw std::runtime_error("LiveUpdate: Restore::as_int() encountered incorrect type " + std::to_string(ent->type)); } std::string Restore::as_string() const { if (ent->type == TYPE_STRING) return std::string(ent->data(), ent->len); - throw std::runtime_error("LiveUpdate: Incorrect type " + std::to_string(ent->type)); + throw std::runtime_error("LiveUpdate: Restore::as_string() encountered incorrect type " + std::to_string(ent->type)); } buffer_t Restore::as_buffer() const { @@ -138,7 +138,7 @@ buffer_t Restore::as_buffer() const buffer.assign(ent->data(), ent->data() + ent->len); return buffer; } - throw std::runtime_error("LiveUpdate: Incorrect type " + std::to_string(ent->type)); + throw std::runtime_error("LiveUpdate: Restore::as_buffer() encountered incorrect type " + std::to_string(ent->type)); } int16_t Restore::get_type() const noexcept @@ -160,11 +160,11 @@ const void* Restore::data() const noexcept const void* Restore::get_segment(size_t size, size_t& count) const { if (ent->type != TYPE_VECTOR) - throw std::runtime_error("LiveUpdate: Incorrect type " + std::to_string(ent->type)); + throw std::runtime_error("LiveUpdate: Restore::as_vector() encountered incorrect type " + std::to_string(ent->type)); auto& segs = ent->get_segs(); if (size != segs.esize) - throw std::runtime_error("LiveUpdate: Incorrect type size " + std::to_string(size) + " vs " + std::to_string(segs.esize)); + throw std::runtime_error("LiveUpdate: Restore::as_vector() encountered incorrect type size " + std::to_string(size) + " vs " + std::to_string(segs.esize)); count = segs.count; return (const void*) segs.vla; @@ -172,7 +172,7 @@ const void* Restore::get_segment(size_t size, size_t& count) const std::vector Restore::rebuild_string_vector() const { if (ent->type != TYPE_STR_VECTOR) - throw std::runtime_error("LiveUpdate: Incorrect type " + std::to_string(ent->type)); + throw std::runtime_error("LiveUpdate: Restore::as_vector() encountered incorrect type " + std::to_string(ent->type)); std::vector retv; // reserve just enough room auto* begin = (varseg_begin*) ent->vla; @@ -193,7 +193,7 @@ std::vector Restore::rebuild_string_vector() const void Restore::go_next() { if (is_end()) - throw std::out_of_range("Already reached end of partition"); + throw std::out_of_range("Restore::go_next(): Already reached end of partition"); // increase the counter, so the resume loop skips entries properly ent = ent->next(); } @@ -219,7 +219,7 @@ void Restore::pop_marker(uint16_t id) && is_end() == false) go_next(); if (is_marker()) { if (get_id() != id) - throw std::out_of_range("Ran past marker with another id: " + std::to_string(get_id())); + throw std::out_of_range("Restore::pop_marker(): Ran past marker with another id: " + std::to_string(get_id())); go_next(); } } From 7f5f3f06586a5f0fa6ba44a6af10fe7d893df8f5 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 26 Sep 2018 15:19:54 +0200 Subject: [PATCH 659/723] statman: Gracefully ignore old serialization method --- src/util/statman_liu.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/util/statman_liu.cpp b/src/util/statman_liu.cpp index d1f1f0aabc..69d76d5676 100644 --- a/src/util/statman_liu.cpp +++ b/src/util/statman_liu.cpp @@ -1,5 +1,7 @@ #include #include +#define TYPE_BUFFER 11 +#define TYPE_VECTOR 12 void Statman::store(uint32_t id, liu::Storage& store) { @@ -7,6 +9,11 @@ void Statman::store(uint32_t id, liu::Storage& store) } void Statman::restore(liu::Restore& store) { + if (store.get_type() != TYPE_VECTOR) { + assert(store.get_type() == TYPE_BUFFER); + // discard old stats that was stored as buffer + return; + } auto stats = store.as_vector(); for (auto& merge_stat : stats) From a63d94314f03d7ea8f74687c2d99f2f2009f14f2 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 26 Sep 2018 16:01:26 +0200 Subject: [PATCH 660/723] kernel: Print plugins during panic, use custom backtrace printer --- api/kernel/syscalls.hpp | 1 + src/kernel/syscalls.cpp | 19 ++++++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/api/kernel/syscalls.hpp b/api/kernel/syscalls.hpp index 9a99bc8182..88731a6c44 100644 --- a/api/kernel/syscalls.hpp +++ b/api/kernel/syscalls.hpp @@ -28,6 +28,7 @@ extern "C" { size_t get_crash_context_length(); } extern void print_backtrace(); +extern void print_backtrace2(void(*stdout_function)(const char*, size_t)); #ifndef SET_CRASH_CONTEXT // used to set a message that will be printed on crash the message is to diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index fb62db06e9..0e95ef39f0 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -143,7 +144,23 @@ void panic(const char* why) (ulong) (OS::heap_end() - OS::heap_begin()) / 1024, (ulong) heap_total / 1024, OS::heap_usage() / 1024); //, total * 100.0); - print_backtrace(); + // print plugins + extern OS::ctor_t __plugin_ctors_start; + extern OS::ctor_t __plugin_ctors_end; + fprintf(stderr, "*** Found %u plugin constructors:\n", + uint32_t(&__plugin_ctors_end - &__plugin_ctors_start)); + for (OS::ctor_t* ptr = &__plugin_ctors_start; ptr < &__plugin_ctors_end; ptr++) + { + char buffer[4096]; + auto res = Elf::safe_resolve_symbol((void*) *ptr, buffer, sizeof(buffer)); + fprintf(stderr, "Plugin: %s (%p)\n", res.name, (void*) res.addr); + } + + // finally, backtrace + fprintf(stderr, "\n*** Backtrace:"); + print_backtrace2([] (const char* text, size_t len) { + fprintf(stderr, "%.*s", (int) len, text); + }); fflush(stderr); SMP::global_unlock(); From 2258b639ca11f4894bcc090effa1621bb7986979 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 26 Sep 2018 22:02:19 +0200 Subject: [PATCH 661/723] kernel: separate liveupdate from heap, improve OOM handling --- api/kernel/os.hpp | 12 +++++++++++- api/util/bitops.hpp | 5 ++++- lib/LiveUpdate/os.cpp | 17 ++++++++++++++--- src/kernel/heap.cpp | 3 +++ src/kernel/os.cpp | 10 ++++++++++ src/kernel/syscalls.cpp | 5 ++++- src/musl/mmap.cpp | 3 ++- src/net/buffer_store.cpp | 8 +++++--- 8 files changed, 53 insertions(+), 10 deletions(-) diff --git a/api/kernel/os.hpp b/api/kernel/os.hpp index 9b63b8d657..6a52d6bfb1 100644 --- a/api/kernel/os.hpp +++ b/api/kernel/os.hpp @@ -203,6 +203,9 @@ class OS { return memory_end_; } + /** Total used memory, including reserved areas */ + static size_t total_memuse() noexcept; + static void init_heap(uintptr_t phys_begin, size_t size) noexcept; /** @@ -211,9 +214,16 @@ class OS { */ static bool is_live_updated() noexcept; - /** Returns the automatic location set aside for storing system and program state **/ + /** Returns the virtual memory location set aside for storing system and program state **/ static void* liveupdate_storage_area() noexcept; + /** Returns the amount of memory set aside for LiveUpdate */ + static size_t liveupdate_phys_size(size_t) noexcept; + + /** Computes the physical location of LiveUpdate storage area */ + static uintptr_t liveupdate_phys_loc(size_t) noexcept; + + /** * A map of memory ranges. The key is the starting address in numeric form. * @note : the idea is to avoid raw pointers whenever possible diff --git a/api/util/bitops.hpp b/api/util/bitops.hpp index ca9f352145..7e3ca27ec8 100644 --- a/api/util/bitops.hpp +++ b/api/util/bitops.hpp @@ -193,7 +193,10 @@ inline bool is_aligned(uintptr_t A, uintptr_t ptr) return (ptr & (A - 1)) == 0; } - +inline size_t upercent(size_t a, size_t b) noexcept +{ + return (100 * a + b / 2) / b; +} } // ns bitops } // ns util diff --git a/lib/LiveUpdate/os.cpp b/lib/LiveUpdate/os.cpp index 1141c1a4ca..86a0d79c11 100644 --- a/lib/LiveUpdate/os.cpp +++ b/lib/LiveUpdate/os.cpp @@ -3,6 +3,10 @@ #include #define HIGHMEM_LOCATION (1ull << 45) +#define LIVEUPDATE_AREA_SIZE 25 +static_assert(LIVEUPDATE_AREA_SIZE > 0 && LIVEUPDATE_AREA_SIZE < 50, + "LIVEUPDATE_AREA_SIZE must be a value between 1 and 50"); + static uintptr_t temp_phys = 0; //#define LIU_DEBUG 1 @@ -12,6 +16,14 @@ static uintptr_t temp_phys = 0; #define PRATTLE(fmt, ...) /* fmt */ #endif +size_t OS::liveupdate_phys_size(size_t phys_max) noexcept { + return phys_max / (100 / size_t(LIVEUPDATE_AREA_SIZE)); +}; + +uintptr_t OS::liveupdate_phys_loc(size_t phys_max) noexcept { + return (phys_max - liveupdate_phys_size(phys_max)) & ~(uintptr_t) 0xFFF; +}; + void OS::setup_liveupdate(uintptr_t phys) { PRATTLE("Setting up LiveUpdate with phys at %p\n", (void*) phys); @@ -27,9 +39,8 @@ void OS::setup_liveupdate(uintptr_t phys) size_t size = 0; if (phys == 0) { - // default is 1/4 of heap from the end of memory - size = OS::heap_max() / 4; - phys = (OS::heap_max() - size) & ~(uintptr_t) 0xFFF; // page aligned + size = OS::liveupdate_phys_size(OS::heap_max()); + phys = OS::liveupdate_phys_loc(OS::heap_max()); } else { size = OS::heap_max() - phys; diff --git a/src/kernel/heap.cpp b/src/kernel/heap.cpp index 99b4e53a98..9899722968 100644 --- a/src/kernel/heap.cpp +++ b/src/kernel/heap.cpp @@ -60,6 +60,9 @@ uintptr_t OS::heap_end() noexcept return mmap_allocation_end(); } +size_t OS::total_memuse() noexcept { + return heap_usage() + OS::liveupdate_phys_size(OS::heap_max()) + heap_begin_; +} constexpr size_t heap_alignment = 4096; __attribute__((weak)) ssize_t __brk_max = 0x100000; diff --git a/src/kernel/os.cpp b/src/kernel/os.cpp index c9731a79a1..f3348cd403 100644 --- a/src/kernel/os.cpp +++ b/src/kernel/os.cpp @@ -80,6 +80,16 @@ void* OS::liveupdate_storage_area() noexcept return (void*) OS::liveupdate_loc_; } +__attribute__((weak)) +size_t OS::liveupdate_phys_size(size_t phys_max) noexcept { + return 4096; +}; + +__attribute__((weak)) +size_t OS::liveupdate_phys_loc(size_t phys_max) noexcept { + return phys_max - liveupdate_phys_size(phys_max); +}; + __attribute__((weak)) void OS::setup_liveupdate(uintptr_t) { diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index fb62db06e9..e7bd4d7db5 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #if defined (UNITTESTS) && !defined(__MACH__) #define THROW throw() @@ -139,9 +140,11 @@ void panic(const char* why) uintptr_t heap_total = OS::heap_max() - OS::heap_begin(); fprintf(stderr, "Heap is at: %p / %p (diff=%lu)\n", (void*) OS::heap_end(), (void*) OS::heap_max(), (ulong) (OS::heap_max() - OS::heap_end())); - fprintf(stderr, "Heap area: %lu / %lu Kb (allocated %zu kb)\n\n", // (%.2f%%)\n", + fprintf(stderr, "Heap area: %lu / %lu Kb (allocated %zu kb)\n", // (%.2f%%)\n", (ulong) (OS::heap_end() - OS::heap_begin()) / 1024, (ulong) heap_total / 1024, OS::heap_usage() / 1024); //, total * 100.0); + fprintf(stderr, "Total memory use: ~%zu%% (%zu of %zu b)\n", + util::bits::upercent(OS::total_memuse(), OS::memory_end()), OS::total_memuse(), OS::memory_end()); print_backtrace(); fflush(stderr); diff --git a/src/musl/mmap.cpp b/src/musl/mmap.cpp index ad27dd9fb4..07cfb26c8a 100644 --- a/src/musl/mmap.cpp +++ b/src/musl/mmap.cpp @@ -18,7 +18,8 @@ Alloc& os::mem::allocator() { uintptr_t __init_mmap(uintptr_t addr_begin) { auto aligned_begin = (addr_begin + Alloc::align - 1) & ~(Alloc::align - 1); - int64_t len = (OS::heap_max() - aligned_begin) & ~int64_t(Alloc::align - 1); + auto mem_end = OS::liveupdate_phys_loc(OS::heap_max()); + int64_t len = (mem_end - aligned_begin) & ~int64_t(Alloc::align - 1); alloc = Alloc::create((void*)aligned_begin, len); kprintf("* mmap initialized. Begin: 0x%zx, end: 0x%zx\n", diff --git a/src/net/buffer_store.cpp b/src/net/buffer_store.cpp index a439b8fe94..85e6a92434 100644 --- a/src/net/buffer_store.cpp +++ b/src/net/buffer_store.cpp @@ -65,7 +65,7 @@ namespace net { else throw std::runtime_error("This BufferStore has run out of buffers"); } - + auto* addr = available_.back(); available_.pop_back(); BSD_PRINT("%d: Gave away %p, %zu buffers remain\n", @@ -76,7 +76,9 @@ namespace net { void BufferStore::create_new_pool() { auto* pool = (uint8_t*) aligned_alloc(OS::page_size(), poolsize_); - assert(pool != nullptr); + if (UNLIKELY(pool == nullptr)) { + throw std::runtime_error("Buffer store failed to allocate memory"); + } this->pools_.push_back(pool); for (uint8_t* b = pool; b < pool + poolsize_; b += bufsize_) { @@ -90,7 +92,7 @@ namespace net { { // TODO: hmm } - + __attribute__((weak)) bool BufferStore::growth_enabled() const { return true; From 8329ba4f3e98c0a47470479feb61445b44c54d07 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Wed, 26 Sep 2018 22:27:32 +0200 Subject: [PATCH 662/723] test: add OS::total_memuse() and memory_end_ to os_mock --- test/lest_util/os_mock.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index f34fe388cc..fabd23790e 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -218,8 +218,11 @@ uintptr_t __brk_max = 0; uintptr_t OS::heap_begin() noexcept { return 0; } + +uintptr_t OS::memory_end_ = 1 << 30; + uintptr_t OS::heap_end() noexcept { - return 1 << 30; + return memory_end_; } size_t OS::heap_usage() noexcept { @@ -230,4 +233,8 @@ uintptr_t OS::heap_max() noexcept { return -1; } +size_t OS::total_memuse() noexcept { + return heap_end(); +} + #endif From 95bd5f4131e57157386dddd150475ccd7fb91ece Mon Sep 17 00:00:00 2001 From: taiyeba Date: Fri, 21 Sep 2018 12:53:49 +0200 Subject: [PATCH 663/723] Build Docker Image: from IncludeOS repo --- .dockerignore | 9 ++++ Dockerfile | 117 +++++++++++++++++++++++++++++++++++++++-------- README_docker.md | 61 ++++++++++++++++++++++++ 3 files changed, 168 insertions(+), 19 deletions(-) create mode 100644 .dockerignore create mode 100644 README_docker.md diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..dcd97e955e --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +build_* +IncludeOS_install/ +diskimagebuild/build +vmbuild/build +src/chainload/build +test/ +examples/ +*/build/ +*/*/build/ diff --git a/Dockerfile b/Dockerfile index 73bc4c3570..2db6584896 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,110 @@ -FROM ubuntu:xenial +FROM ubuntu:xenial as base RUN apt-get update && apt-get -y install \ - git \ - net-tools \ sudo \ - wget \ -&& rm -rf /var/lib/apt/lists/* - -RUN useradd --create-home -s /bin/bash ubuntu -RUN adduser ubuntu sudo -RUN echo -n 'ubuntu:ubuntu' | chpasswd + curl \ + locales \ + && rm -rf /var/lib/apt/lists/* +RUN locale-gen en_US.UTF-8 +ENV LANG=en_US.UTF-8 \ + LANGUAGE=en_US:en \ + LC_ALL=en_US.UTF-8 -# Enable passwordless sudo for users under the "sudo" group -RUN sed -i.bkp -e \ +# Add fixuid to change permissions for bind-mounts. Set uid to same as host with -u : +RUN addgroup --gid 1000 docker && \ + adduser --uid 1000 --ingroup docker --home /home/docker --shell /bin/sh --disabled-password --gecos "" docker && \ + usermod -aG sudo docker && \ + sed -i.bkp -e \ 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' \ /etc/sudoers +RUN USER=docker && \ + GROUP=docker && \ + curl -SsL https://github.com/boxboat/fixuid/releases/download/v0.3/fixuid-0.3-linux-amd64.tar.gz | tar -C /usr/local/bin -xzf - && \ + chown root:root /usr/local/bin/fixuid && \ + chmod 4755 /usr/local/bin/fixuid && \ + mkdir -p /etc/fixuid && \ + printf "user: $USER\ngroup: $GROUP\npaths:\n - /service\n" > /etc/fixuid/config.yml -USER ubuntu +RUN echo "LANG=C.UTF-8" > /etc/default/locale -ADD . /home/ubuntu/IncludeOS -WORKDIR /home/ubuntu/IncludeOS +# Docker TAG can be specified when building with --build-arg TAG=..., this is redeclared in the source-build stage +# Git tags can be specified with --build-arg NEWTAG=..., the default is set as includeos-dev +ARG BRANCH=dev +ARG REPO=hioa-cs +ARG NEWTAG=includeos-dev +ENV BRANCH=$BRANCH +ENV REPO=$REPO +ENV NEWTAG=$NEWTAG -RUN sudo apt-get update && \ - sudo do_bridge="" ./etc/install_all_source.sh \ -&& sudo rm -rf /var/lib/apt/lists/* -VOLUME /service +LABEL dockerfile.version=1 \ + includeos.version=$BRANCH WORKDIR /service -CMD ./run.sh +######################### +FROM base as source-build + +RUN apt-get update && apt-get -y install \ + git \ + lsb-release \ + net-tools \ + wget \ + && rm -rf /var/lib/apt/lists/* + + +# Copy IncludeOS contents from host repo to container +RUN echo "copying Contents of current IncludeOS branch" +RUN cd ~ && pwd && \ + mkdir -p IncludeOS +COPY . /root/IncludeOS/ + +# Adding Custom git tags +RUN echo "Assigning Your custom git tag" +RUN cd /root/IncludeOS && \ + git tag -d $(git describe --tags) ; git tag $NEWTAG + +# Installation +RUN cd /root/IncludeOS && \ + ./install.sh -n + +RUN git -C /root/IncludeOS describe --dirty --tags > /ios_version.txt + +############################# +FROM base as grubify + +RUN apt-get update && apt-get -y install \ + dosfstools \ + grub-pc + +COPY --from=source-build /usr/local/includeos/scripts/grubify.sh /home/ubuntu/IncludeOS_install/includeos/scripts/grubify.sh + +ENTRYPOINT ["fixuid", "/home/ubuntu/IncludeOS_install/includeos/scripts/grubify.sh"] + +########################### +FROM base as build + +RUN apt-get update && apt-get -y install \ + git \ + clang-5.0 \ + cmake \ + nasm \ + python-pip \ + && rm -rf /var/lib/apt/lists/* \ + && pip install pystache antlr4-python2-runtime && \ + apt-get remove -y python-pip && \ + apt autoremove -y + +COPY --from=source-build /usr/local/includeos /usr/local/includeos/ +COPY --from=source-build /usr/local/bin/boot /usr/local/bin/boot +COPY --from=source-build /root/IncludeOS/etc/install_dependencies_linux.sh / +COPY --from=source-build /root/IncludeOS/etc/use_clang_version.sh / +COPY --from=source-build /root/IncludeOS/lib/uplink/starbase /root/IncludeOS/lib/uplink/starbase/ +COPY --from=source-build /ios_version.txt / +COPY --from=source-build /root/IncludeOS/etc/docker_entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] + +CMD mkdir -p build && \ + cd build && \ + cp $(find /usr/local/includeos -name chainloader) /service/build/chainloader && \ + cmake .. && \ + make diff --git a/README_docker.md b/README_docker.md new file mode 100644 index 0000000000..e1fea327c0 --- /dev/null +++ b/README_docker.md @@ -0,0 +1,61 @@ + +# Docker images + +These Docker images let you try out building [IncludeOS](https://github.com/hioa-cs/IncludeOS/) unikernels without having to install the development environment locally on your machine. + +## Build options +When building the docker image there are a few options available: + +|Action| Command | +|--|--| +|Specify build version| ```--build-arg TAG=``` | +|Service to build|```--target=```| + +### Docker tag structure +The docker tags in use for these images are: +``` +/:. +includeos/build:v0.12.0-rc.3.1 +``` +For every change made to the dockerfile the corresponding tag is incremented. + +## Building services +``` +$ docker build --build-arg TAG=v0.12.0-rc.3 --target=build -t includeos/build:v0.12.0-rc.3.01 . +$ cd +$ docker run --rm -v $PWD:/service includeos/build:v0.12.0-rc.3.01 +``` + +## Adding a GRUB bootloader to your service +On macOS, the `boot -g` option to add a GRUB bootloader is not available. Instead, you can use the `includeos/grubify` Docker image. +``` +$ docker build --build-arg TAG=v0.12.0-rc.3 --target=grubify -t includeos/grubify:v0.12.0-rc.3.01 . +$ docker run --rm --privileged -v $PWD:/service includeos/grubify:v0.12.0-rc.3.01 build/ +``` + +## Running a very basic sanity test of your service image + +Don't have a hypervisor installed? No problem? Run your service inside QEMU in a Docker container: + +``` +$ docker run --rm -v $PWD:/service/build includeos/includeos-qemu:v0.12.0-rc.2.0 +``` + +(If the service is not designed to exit on its own, the container must be stopped with `docker stop`.) + + +## Building a tiny web server + +Do you have some web content that you would like to serve, without having to figure out arcane Apache configuration files? Just go to the folder where your web content is located and build a bootable web server: + +``` +$ docker build --build-arg TAG=v0.12.0-rc.3 --target=webserver -t includeos/webserver:v0.12.0-rc.3.01 . +docker run --rm -v $PWD:/public includeos/webserver:v0.12.0-rc.3.01 +``` + +## FAQ +### Specify user field to prevent permissions errors +When using bind mounts in docker there are potential errors with user permissions of the files that are mounted. This has been seen on linux systems where a non default uid/gid was used. To work around this we added [Fixuid](https://github.com/boxboat/fixuid) to the build and grubify images. This ensures that the build/grubify container has the correct user permissions to add/modify the required mounted files. If user is not specified then the docker containers use the root user. To specify user add your uid/gid to the docker options: +``` +--user $(id -u):$(id -g) +``` From 67b57821f15e8ec56f657c5afd0c38983bef4357 Mon Sep 17 00:00:00 2001 From: taiyeba Date: Thu, 27 Sep 2018 10:53:06 +0200 Subject: [PATCH 664/723] Build Docker Image: Optimizing code and ensuring custom tag works Co-authored-by: Martin Nordsletten martin@includeos.org --- .dockerignore | 3 +-- Dockerfile | 38 +++++++++++--------------------------- etc/docker_entrypoint.sh | 20 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 29 deletions(-) create mode 100644 etc/docker_entrypoint.sh diff --git a/.dockerignore b/.dockerignore index dcd97e955e..ce5d790199 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,5 +5,4 @@ vmbuild/build src/chainload/build test/ examples/ -*/build/ -*/*/build/ +**/build diff --git a/Dockerfile b/Dockerfile index 2db6584896..e8042aae03 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ FROM ubuntu:xenial as base +LABEL maintainer="Martin Nordsletten, S Taiyeba Haroon" RUN apt-get update && apt-get -y install \ sudo \ @@ -27,21 +28,9 @@ RUN USER=docker && \ RUN echo "LANG=C.UTF-8" > /etc/default/locale -# Docker TAG can be specified when building with --build-arg TAG=..., this is redeclared in the source-build stage -# Git tags can be specified with --build-arg NEWTAG=..., the default is set as includeos-dev -ARG BRANCH=dev -ARG REPO=hioa-cs -ARG NEWTAG=includeos-dev -ENV BRANCH=$BRANCH -ENV REPO=$REPO -ENV NEWTAG=$NEWTAG - - -LABEL dockerfile.version=1 \ - includeos.version=$BRANCH -WORKDIR /service ######################### + FROM base as source-build RUN apt-get update && apt-get -y install \ @@ -51,25 +40,19 @@ RUN apt-get update && apt-get -y install \ wget \ && rm -rf /var/lib/apt/lists/* +RUN mkdir -p /root/IncludeOS +WORKDIR /root/IncludeOS +COPY . . -# Copy IncludeOS contents from host repo to container -RUN echo "copying Contents of current IncludeOS branch" -RUN cd ~ && pwd && \ - mkdir -p IncludeOS -COPY . /root/IncludeOS/ - -# Adding Custom git tags -RUN echo "Assigning Your custom git tag" -RUN cd /root/IncludeOS && \ - git tag -d $(git describe --tags) ; git tag $NEWTAG +# Ability to specify custom tag that overwrites any existing tag. This will then match the reported IncludeOS version +ARG TAG +RUN : ${TAG:=$(git describe --tags)} && git tag -d $(git describe --tags); git tag $TAG && git describe --tags --dirty > /ios_version.txt # Installation -RUN cd /root/IncludeOS && \ - ./install.sh -n - -RUN git -C /root/IncludeOS describe --dirty --tags > /ios_version.txt +RUN ./install.sh -n ############################# + FROM base as grubify RUN apt-get update && apt-get -y install \ @@ -81,6 +64,7 @@ COPY --from=source-build /usr/local/includeos/scripts/grubify.sh /home/ubuntu/In ENTRYPOINT ["fixuid", "/home/ubuntu/IncludeOS_install/includeos/scripts/grubify.sh"] ########################### + FROM base as build RUN apt-get update && apt-get -y install \ diff --git a/etc/docker_entrypoint.sh b/etc/docker_entrypoint.sh new file mode 100644 index 0000000000..bcd4eaf3d1 --- /dev/null +++ b/etc/docker_entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# entrypoint.sh exports the correct clang version + +# fixuid gives the user in the container the same uid:gid as specified when running +# docker run with --user uid:gid. This is to prevent file permission errors +eval $( fixuid &> /dev/null ) + +pushd / > /dev/null +if version=$(grep -oP 'CLANG_VERSION_MIN_REQUIRED="\K[^"]+' use_clang_version.sh); then : +elif version=$(grep -oP 'CLANG_VERSION_MIN_REQUIRED="\K[^"]+' install_dependencies_linux.sh); then : +elif version=$(grep -oP 'CLANG_VERSION="\K[^"]+' install_dependencies_linux.sh); then : +else version=3.8 +fi +export CC=clang-$version +export CXX=clang++-$version +popd > /dev/null + +# Execute any command following entrypoint.sh +exec "$@" From d4750c4957afe0c71c92a461a67358521fed95da Mon Sep 17 00:00:00 2001 From: taiyeba Date: Thu, 27 Sep 2018 11:52:22 +0200 Subject: [PATCH 665/723] Build Docker Image: Specify working directory --- Dockerfile | 2 ++ etc/docker_entrypoint.sh | 0 2 files changed, 2 insertions(+) mode change 100644 => 100755 etc/docker_entrypoint.sh diff --git a/Dockerfile b/Dockerfile index e8042aae03..b19aae63dd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,6 +78,8 @@ RUN apt-get update && apt-get -y install \ apt-get remove -y python-pip && \ apt autoremove -y +WORKDIR /service + COPY --from=source-build /usr/local/includeos /usr/local/includeos/ COPY --from=source-build /usr/local/bin/boot /usr/local/bin/boot COPY --from=source-build /root/IncludeOS/etc/install_dependencies_linux.sh / diff --git a/etc/docker_entrypoint.sh b/etc/docker_entrypoint.sh old mode 100644 new mode 100755 From cbbafbee5bc41d27e5f1247537674c572b158d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Jerpetj=C3=B8n?= Date: Thu, 27 Sep 2018 13:09:16 +0200 Subject: [PATCH 666/723] Removed the feature from vmxnet3 that tells the vmware driver to strip vlan tags --- src/drivers/vmxnet3.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index 9d0aa007d6..da1c4baafd 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -1,4 +1,4 @@ -// This file is a part of the IncludeOS unikernel - www.includeos.org +// This file is a part of the IncludeOS unikernel - www.includeos.org // // Copyright 2015 Oslo and Akershus University College of Applied Sciences // and Alfred Bratterud @@ -237,7 +237,7 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : shared.misc.version = VMXNET3_VERSION_MAGIC; shared.misc.version_support = 1; shared.misc.upt_version_support = 1; - shared.misc.upt_features = UPT1_F_RXVLAN; + shared.misc.upt_features = 0; shared.misc.driver_data_address = (uintptr_t) &dma; shared.misc.queue_desc_address = (uintptr_t) &dma->queues; shared.misc.driver_data_len = sizeof(vmxnet3_dma); From 395427de69e421d6bb55db2397feb06fcd8e4c8f Mon Sep 17 00:00:00 2001 From: taiyeba Date: Thu, 27 Sep 2018 14:18:15 +0200 Subject: [PATCH 667/723] Build Docker Image: removing docker README --- Dockerfile | 1 - README_docker.md | 61 ------------------------------------------------ 2 files changed, 62 deletions(-) delete mode 100644 README_docker.md diff --git a/Dockerfile b/Dockerfile index b19aae63dd..617087b4e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,6 @@ RUN USER=docker && \ RUN echo "LANG=C.UTF-8" > /etc/default/locale - ######################### FROM base as source-build diff --git a/README_docker.md b/README_docker.md deleted file mode 100644 index e1fea327c0..0000000000 --- a/README_docker.md +++ /dev/null @@ -1,61 +0,0 @@ - -# Docker images - -These Docker images let you try out building [IncludeOS](https://github.com/hioa-cs/IncludeOS/) unikernels without having to install the development environment locally on your machine. - -## Build options -When building the docker image there are a few options available: - -|Action| Command | -|--|--| -|Specify build version| ```--build-arg TAG=``` | -|Service to build|```--target=```| - -### Docker tag structure -The docker tags in use for these images are: -``` -/:. -includeos/build:v0.12.0-rc.3.1 -``` -For every change made to the dockerfile the corresponding tag is incremented. - -## Building services -``` -$ docker build --build-arg TAG=v0.12.0-rc.3 --target=build -t includeos/build:v0.12.0-rc.3.01 . -$ cd -$ docker run --rm -v $PWD:/service includeos/build:v0.12.0-rc.3.01 -``` - -## Adding a GRUB bootloader to your service -On macOS, the `boot -g` option to add a GRUB bootloader is not available. Instead, you can use the `includeos/grubify` Docker image. -``` -$ docker build --build-arg TAG=v0.12.0-rc.3 --target=grubify -t includeos/grubify:v0.12.0-rc.3.01 . -$ docker run --rm --privileged -v $PWD:/service includeos/grubify:v0.12.0-rc.3.01 build/ -``` - -## Running a very basic sanity test of your service image - -Don't have a hypervisor installed? No problem? Run your service inside QEMU in a Docker container: - -``` -$ docker run --rm -v $PWD:/service/build includeos/includeos-qemu:v0.12.0-rc.2.0 -``` - -(If the service is not designed to exit on its own, the container must be stopped with `docker stop`.) - - -## Building a tiny web server - -Do you have some web content that you would like to serve, without having to figure out arcane Apache configuration files? Just go to the folder where your web content is located and build a bootable web server: - -``` -$ docker build --build-arg TAG=v0.12.0-rc.3 --target=webserver -t includeos/webserver:v0.12.0-rc.3.01 . -docker run --rm -v $PWD:/public includeos/webserver:v0.12.0-rc.3.01 -``` - -## FAQ -### Specify user field to prevent permissions errors -When using bind mounts in docker there are potential errors with user permissions of the files that are mounted. This has been seen on linux systems where a non default uid/gid was used. To work around this we added [Fixuid](https://github.com/boxboat/fixuid) to the build and grubify images. This ensures that the build/grubify container has the correct user permissions to add/modify the required mounted files. If user is not specified then the docker containers use the root user. To specify user add your uid/gid to the docker options: -``` ---user $(id -u):$(id -g) -``` From e9f0f92dd243a2063e01c6a16bc8d66e217b1c6e Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 28 Sep 2018 12:11:28 +0200 Subject: [PATCH 668/723] buddy: take overbooking into account more places + fixes to test --- api/util/alloc_buddy.hpp | 11 +++++++++-- test/util/unit/buddy_alloc_test.cpp | 24 +++++++++++++++++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index c0c78fb702..3e47b2d1e4 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -218,8 +218,12 @@ namespace os::mem::buddy { return root().is_full(); } + bool overbooked() { + return overbooked_; + } + bool empty() { - return root().is_free(); + return bytes_used() == 0; } Addr_t addr_begin() const noexcept { @@ -332,8 +336,10 @@ namespace os::mem::buddy { // For overbooking allocator, allow unusable memory to gradually become // marked as allocated, without actually handing it out. - if (UNLIKELY(res + size > addr_limit_)) + if (UNLIKELY(res + size > addr_limit_)) { + overbooked_ = true; return 0; + } if (res) bytes_used_ += sz; return reinterpret_cast(res); @@ -732,6 +738,7 @@ namespace os::mem::buddy { const uintptr_t addr_limit_ = 0; const Size_t pool_size_ = min_size; Size_t bytes_used_ = 0; + bool overbooked_ = false; }; /** diff --git a/test/util/unit/buddy_alloc_test.cpp b/test/util/unit/buddy_alloc_test.cpp index b80109ddd9..888d5a80ad 100644 --- a/test/util/unit/buddy_alloc_test.cpp +++ b/test/util/unit/buddy_alloc_test.cpp @@ -30,7 +30,10 @@ struct Pool { Pool(size_t s) : size{s} { auto sz = Alloc::max_bufsize(s); auto res = posix_memalign(&addr, Alloc::min_size, sz); - Expects(res == 0); + if (res != 0) { + printf("Failed to allocate memory for allocator\n"); + Expects(res != 0); + } alloc = Alloc::create

(addr, sz); } @@ -147,7 +150,7 @@ CASE("mem::buddy random ordered allocation then deallocation"){ if (not alloc.bytes_free()) break; - const auto sz = alloc.chunksize(rnd % std::max(size_t(32_KiB), pool.size / 1024)); + const auto sz = alloc.chunksize(rnd % std::max(size_t(32_KiB), pool.size / test::random_1k.size())); #ifdef DEBUG_UNIT std::cout << "Alloc " << Byte_r(sz) << "\n"; @@ -160,6 +163,9 @@ CASE("mem::buddy random ordered allocation then deallocation"){ } auto addr = alloc.allocate(sz); EXPECT(addr); + if (addr == 0) { + continue; + } EXPECT(alloc.in_range(addr)); addresses.push_back(addr); sizes.push_back(sz); @@ -189,7 +195,7 @@ CASE("mem::buddy random ordered allocation then deallocation"){ } uintptr_t hi_used = (uintptr_t)addresses.at(highest_i) + sizes.at(highest_i); - EXPECT(hi_used == alloc.highest_used()); + EXPECT((hi_used == alloc.highest_used() or alloc.overbooked())); auto computed_use = hi_used - alloc.addr_begin(); EXPECT(computed_use >= alloc.bytes_used()); @@ -237,7 +243,7 @@ struct Allocation { bool overlaps(Allocation other){ return overlaps(other.addr_begin()) - or overlaps(other.addr_end()); + or overlaps(other.addr_end() - 1); } bool verify_addr() { @@ -256,6 +262,9 @@ struct Allocation { } }; +std::ostream& operator<<(std::ostream& out, Allocation& a) { + return out << "[ " << a.addr << ", " << a.addr + a.size << " ]"; +} CASE("mem::buddy random chaos with data verification"){ using namespace util; @@ -265,17 +274,21 @@ CASE("mem::buddy random chaos with data verification"){ EXPECT(bool(alloc.root())); EXPECT(alloc.bytes_free() == alloc.capacity()); + EXPECT(alloc.empty()); std::vector allocs; for (auto rnd : test::random_1k) { - auto sz = rnd % alloc.pool_size_ / 1024; + auto sz = std::max(rnd % alloc.pool_size_ / 1024, alloc.min_size); EXPECT(sz); if (not alloc.full()) { Allocation a{&alloc}; a.size = sz; a.addr = (uintptr_t)alloc.allocate(sz); + if (a.addr == 0) { + continue; + } a.data = 'A' + (rnd % ('Z' - 'A')); EXPECT(a.addr); EXPECT(a.verify_addr()); @@ -284,6 +297,7 @@ CASE("mem::buddy random chaos with data verification"){ std::find_if(allocs.begin(), allocs.end(), [&a](Allocation& other) { return other.overlaps(a); }); + EXPECT(overlap == allocs.end()); allocs.emplace_back(std::move(a)); memset((void*)a.addr, a.data, a.size); From f94c39ba86c6603ed5aeb0cc4f0f1a0584e0964d Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 28 Sep 2018 12:23:55 +0200 Subject: [PATCH 669/723] buddy: take overbooking into account when reporting highest_used --- api/util/alloc_buddy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/util/alloc_buddy.hpp b/api/util/alloc_buddy.hpp index 3e47b2d1e4..e50a0cd0b0 100644 --- a/api/util/alloc_buddy.hpp +++ b/api/util/alloc_buddy.hpp @@ -199,7 +199,7 @@ namespace os::mem::buddy { } Addr_t highest_used() const noexcept { - return root().highest_used_r(); + return std::min(root().highest_used_r(), start_addr_ + capacity()); } Size_t bytes_free() const noexcept { From be6c83256c1a554d8386d14f071ac66b54e33c4b Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Fri, 28 Sep 2018 16:36:19 +0200 Subject: [PATCH 670/723] Updating NaCl commit (Fixing memory calculation (Timer)) --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index 1e338e80e7..f817610bd8 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit 1e338e80e7b8b9839da74972a58ec9da68321d50 +Subproject commit f817610bd8aed8db392a3f889b0f2d8d46ee8214 From 1c4d35f155de4f32cebe85193599423a48584369 Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Fri, 28 Sep 2018 16:46:23 +0200 Subject: [PATCH 671/723] Updating NaCl commit (goldenfiles/timers.cpp updated) --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index f817610bd8..ab869f7a0f 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit f817610bd8aed8db392a3f889b0f2d8d46ee8214 +Subproject commit ab869f7a0fd0942fbb808edd1bb376481e1f866f From e268a79e70d66ae681c394fa2206d7b9b7d44cb0 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Mon, 1 Oct 2018 10:38:39 +0200 Subject: [PATCH 672/723] vmxnet3: Unannounce second RX queue --- src/drivers/vmxnet3.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index da1c4baafd..b09afaeee7 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -212,7 +212,7 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : { memset(rx[q].buffers, 0, sizeof(rx[q].buffers)); rx[q].desc0 = &dma->rx[q].desc[0]; - rx[q].desc1 = &dma->rx[q].desc[0]; + rx[q].desc1 = nullptr; rx[q].comp = &dma->rx[q].comp[0]; rx[q].index = q; @@ -221,7 +221,7 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : queue.cfg.desc_address[1] = (uintptr_t) rx[q].desc1; queue.cfg.comp_address = (uintptr_t) rx[q].comp; queue.cfg.num_desc[0] = vmxnet3::NUM_RX_DESC; - queue.cfg.num_desc[1] = vmxnet3::NUM_RX_DESC; + queue.cfg.num_desc[1] = 0; queue.cfg.num_comp = VMXNET3_NUM_RX_COMP; queue.cfg.driver_data_len = sizeof(vmxnet3_rx_desc) + 2 * sizeof(vmxnet3_rx_desc); From 5f167eb2749556354a7eea7718393acc02018484 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 1 Oct 2018 14:12:22 +0200 Subject: [PATCH 673/723] net: Added packet debugging hook to print latest packet on crash --- api/net/link_layer.hpp | 7 +++- src/CMakeLists.txt | 1 + src/kernel/syscalls.cpp | 7 ++++ src/net/packet_debug.cpp | 84 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 src/net/packet_debug.cpp diff --git a/api/net/link_layer.hpp b/api/net/link_layer.hpp index dc1acc22f6..37106a0bd6 100644 --- a/api/net/link_layer.hpp +++ b/api/net/link_layer.hpp @@ -23,6 +23,8 @@ namespace net { +extern void set_last_packet(net::Packet*); + template class Link_layer : public hw::Nic { public: @@ -81,7 +83,10 @@ class Link_layer : public hw::Nic { protected: /** Called by the underlying physical driver inheriting the Link_layer */ void receive(net::Packet_ptr pkt) - { link_.receive(std::move(pkt)); } + { + set_last_packet(pkt.get()); + link_.receive(std::move(pkt)); + } private: Protocol link_; diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5fde7f2424..0db8c89958 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -63,6 +63,7 @@ set(OS_OBJECTS net/http/server_connection.cpp net/http/server.cpp net/http/response_writer.cpp net/ws/websocket.cpp ${OPENSSL_MODULES} ${BOTAN_MODULES} net/nat/nat.cpp net/nat/napt.cpp + net/packet_debug.cpp fs/disk.cpp fs/filesystem.cpp fs/dirent.cpp fs/mbr.cpp fs/path.cpp fs/fat.cpp fs/fat_async.cpp fs/fat_sync.cpp fs/memdisk.cpp # POSIX diff --git a/src/kernel/syscalls.cpp b/src/kernel/syscalls.cpp index fb62db06e9..fc9b85c361 100644 --- a/src/kernel/syscalls.cpp +++ b/src/kernel/syscalls.cpp @@ -102,6 +102,10 @@ extern "C" __attribute__((noreturn)) void panic_epilogue(const char*); extern "C" __attribute__ ((weak)) void panic_perform_inspection_procedure() {} +namespace net { + __attribute__((weak)) void print_last_packet() {} +} + /** * panic: * Display reason for kernel panic @@ -143,6 +147,9 @@ void panic(const char* why) (ulong) (OS::heap_end() - OS::heap_begin()) / 1024, (ulong) heap_total / 1024, OS::heap_usage() / 1024); //, total * 100.0); + // last packet + net::print_last_packet(); + print_backtrace(); fflush(stderr); SMP::global_unlock(); diff --git a/src/net/packet_debug.cpp b/src/net/packet_debug.cpp new file mode 100644 index 0000000000..ec3acc49e5 --- /dev/null +++ b/src/net/packet_debug.cpp @@ -0,0 +1,84 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2018 IncludeOS AS, Oslo, Norway +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +namespace net { + + static net::Packet* last_packet = nullptr; + static uint8_t* layer_begin = nullptr; + + void set_last_packet(net::Packet* ptr) + { + Expects(ptr != nullptr); + last_packet = ptr; + layer_begin = ptr->layer_begin(); + } + + void print_last_packet() + { + if(last_packet == nullptr) + return; + + auto* pkt = last_packet; + fprintf(stderr, "*** Last packet:\n"); + fprintf(stderr, "Buffer: Begin: %p End: %p Size: %i\n", + pkt->buf(), pkt->buffer_end(), pkt->bufsize()); + fprintf(stderr, "Layer: Recorded: %p Current: %p (%lub offset)\n", + layer_begin, pkt->layer_begin(), pkt->layer_begin() - layer_begin); + fprintf(stderr, "Size: %i ", pkt->size()); + fprintf(stderr, "Capacity: %i ", pkt->capacity()); + fprintf(stderr, "Data end: %p \n", pkt->data_end()); + + // assume ethernet + auto* layer = layer_begin; + Ethernet::header* eth = reinterpret_cast(layer); + fprintf(stderr, "Ethernet type: 0x%hx ", eth->type()); + int print_len = sizeof(Ethernet::header); + switch(eth->type()) + { + case Ethertype::IP4: + fprintf(stderr, "IPv4\n"); + print_len += 20; + break; + case Ethertype::IP6: + fprintf(stderr, "IPv6\n"); + print_len += 40; + break; + case Ethertype::ARP: + fprintf(stderr, "ARP\n"); + print_len += 28; + break; + case Ethertype::VLAN: + fprintf(stderr, "VLAN\n"); + print_len += 4; + break; + default: + fprintf(stderr, "Unknown\n"); + print_len = 0; + } + + // read 40 more optimistic bytes (could be garbage from old packet) + if(pkt->size() > print_len) + print_len += std::min(pkt->size() - print_len, 40); + + fprintf(stderr, "Payload %i bytes from recorded layer begin (%p):\n", print_len, layer_begin); + for(int i = 0; i < print_len; i++) + fprintf(stderr, "%02x", *(layer_begin + i)); + fprintf(stderr, "\n"); + } +} From 32a2c0d3ad466b70409d7d315b716f81faded731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 1 Oct 2018 14:37:47 +0200 Subject: [PATCH 674/723] linux: Add packet_debug to cmakelist --- linux/userspace/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/linux/userspace/CMakeLists.txt b/linux/userspace/CMakeLists.txt index 83faba69c3..77f5ce7d58 100644 --- a/linux/userspace/CMakeLists.txt +++ b/linux/userspace/CMakeLists.txt @@ -6,6 +6,7 @@ set(NET_SOURCES ${IOS}/src/net/configure.cpp ${IOS}/src/net/super_stack.cpp ${IOS}/src/net/inet.cpp + ${IOS}/src/net/packet_debug.cpp ${IOS}/src/net/ethernet/ethernet.cpp ${IOS}/src/net/ip4/arp.cpp From f38d6aa1243cd18835e7808c59ebae2feaea16fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 1 Oct 2018 14:46:43 +0200 Subject: [PATCH 675/723] mana: Calculate a stat's index correctly --- .../include/mana/components/dashboard/components/statman.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mana/include/mana/components/dashboard/components/statman.hpp b/lib/mana/include/mana/components/dashboard/components/statman.hpp index 4dc872b865..91a563c68d 100644 --- a/lib/mana/include/mana/components/dashboard/components/statman.hpp +++ b/lib/mana/include/mana/components/dashboard/components/statman.hpp @@ -62,7 +62,7 @@ class Statman : public Component { writer.String(type); writer.Key("index"); - writer.Int(&stat - statman_.begin()); + writer.Int(std::distance(statman_.begin(), it)); writer.EndObject(); } From 9ba16d150834eb23bcaaf94777eb227385ceeac0 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Tue, 2 Oct 2018 09:04:54 +0200 Subject: [PATCH 676/723] Dockerfile: Save original git describe as well, fixed dockerignore --- .dockerignore | 5 ----- Dockerfile | 4 +++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.dockerignore b/.dockerignore index ce5d790199..12b69947dc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,3 @@ build_* IncludeOS_install/ -diskimagebuild/build -vmbuild/build -src/chainload/build -test/ -examples/ **/build diff --git a/Dockerfile b/Dockerfile index 617087b4e7..4d741591ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,8 @@ COPY . . # Ability to specify custom tag that overwrites any existing tag. This will then match the reported IncludeOS version ARG TAG -RUN : ${TAG:=$(git describe --tags)} && git tag -d $(git describe --tags); git tag $TAG && git describe --tags --dirty > /ios_version.txt +RUN git describe --tags --dirty > /ios_version.txt +RUN : ${TAG:=$(git describe --tags)} && git tag -d $(git describe --tags); git tag $TAG && git describe --tags --dirty > /tag.txt # Installation RUN ./install.sh -n @@ -85,6 +86,7 @@ COPY --from=source-build /root/IncludeOS/etc/install_dependencies_linux.sh / COPY --from=source-build /root/IncludeOS/etc/use_clang_version.sh / COPY --from=source-build /root/IncludeOS/lib/uplink/starbase /root/IncludeOS/lib/uplink/starbase/ COPY --from=source-build /ios_version.txt / +COPY --from=source-build /tag.txt / COPY --from=source-build /root/IncludeOS/etc/docker_entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] From 903d2ec038aad95e18e66b0b4c3dfb3a29ad0223 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Tue, 2 Oct 2018 09:08:17 +0200 Subject: [PATCH 677/723] Gitignore: Added starbase/disk and nacl.txt --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a050b2c304..17d9b39aeb 100644 --- a/.gitignore +++ b/.gitignore @@ -18,6 +18,7 @@ *.img.* *.gz *.tar +*nacl.txt .DS_Store nbproject dummy.disk @@ -47,3 +48,6 @@ IncludeOS_install #CLion .idea/ + +# Starbase disk file +lib/uplink/starbase/disk From 1961d2aba44c539a315e89caf90f6b2799a04028 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Tue, 2 Oct 2018 09:11:53 +0200 Subject: [PATCH 678/723] Docker: Cleaned up entrypoint script. Only keep relevant clang file --- etc/docker_entrypoint.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/etc/docker_entrypoint.sh b/etc/docker_entrypoint.sh index bcd4eaf3d1..44ad8d8127 100755 --- a/etc/docker_entrypoint.sh +++ b/etc/docker_entrypoint.sh @@ -8,9 +8,7 @@ eval $( fixuid &> /dev/null ) pushd / > /dev/null if version=$(grep -oP 'CLANG_VERSION_MIN_REQUIRED="\K[^"]+' use_clang_version.sh); then : -elif version=$(grep -oP 'CLANG_VERSION_MIN_REQUIRED="\K[^"]+' install_dependencies_linux.sh); then : -elif version=$(grep -oP 'CLANG_VERSION="\K[^"]+' install_dependencies_linux.sh); then : -else version=3.8 +else version=5.0 fi export CC=clang-$version export CXX=clang++-$version From 6bd04c95215369b64e73ada01771c41f1bfe8e43 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Tue, 2 Oct 2018 15:48:32 +0200 Subject: [PATCH 679/723] Docker: Changed tag to be custom_tag instead --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4d741591ed..922813efd0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,9 +44,9 @@ WORKDIR /root/IncludeOS COPY . . # Ability to specify custom tag that overwrites any existing tag. This will then match the reported IncludeOS version -ARG TAG +ARG CUSTOM_TAG RUN git describe --tags --dirty > /ios_version.txt -RUN : ${TAG:=$(git describe --tags)} && git tag -d $(git describe --tags); git tag $TAG && git describe --tags --dirty > /tag.txt +RUN : ${CUSTOM_TAG:=$(git describe --tags)} && git tag -d $(git describe --tags); git tag $CUSTOM_TAG && git describe --tags --dirty > /custom_tag.txt # Installation RUN ./install.sh -n @@ -86,7 +86,7 @@ COPY --from=source-build /root/IncludeOS/etc/install_dependencies_linux.sh / COPY --from=source-build /root/IncludeOS/etc/use_clang_version.sh / COPY --from=source-build /root/IncludeOS/lib/uplink/starbase /root/IncludeOS/lib/uplink/starbase/ COPY --from=source-build /ios_version.txt / -COPY --from=source-build /tag.txt / +COPY --from=source-build /custom_tag.txt / COPY --from=source-build /root/IncludeOS/etc/docker_entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] From c5016837d4a4cf1f56466540cbfd3f7d4aa3b756 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 2 Oct 2018 16:22:56 +0200 Subject: [PATCH 680/723] net: Write full packet content on panic --- src/net/packet_debug.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/net/packet_debug.cpp b/src/net/packet_debug.cpp index ec3acc49e5..3ba7cb0059 100644 --- a/src/net/packet_debug.cpp +++ b/src/net/packet_debug.cpp @@ -72,13 +72,14 @@ namespace net { print_len = 0; } - // read 40 more optimistic bytes (could be garbage from old packet) - if(pkt->size() > print_len) - print_len += std::min(pkt->size() - print_len, 40); + // ignore the above, just hope we can write the full content of the packet + print_len = pkt->size(); fprintf(stderr, "Payload %i bytes from recorded layer begin (%p):\n", print_len, layer_begin); - for(int i = 0; i < print_len; i++) + for(int i = 0; i < print_len; i++) { fprintf(stderr, "%02x", *(layer_begin + i)); + if(i > 0 and i % 80 == 0) fprintf(stderr, "\n"); // break every 80th char + } fprintf(stderr, "\n"); } } From 1a6c458e2e27f3b1c5dfa38745721d1ce4c3ba82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 2 Oct 2018 16:36:04 +0200 Subject: [PATCH 681/723] net: Break every 80th char correctly in packet debug --- src/net/packet_debug.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/net/packet_debug.cpp b/src/net/packet_debug.cpp index 3ba7cb0059..b6e095cc79 100644 --- a/src/net/packet_debug.cpp +++ b/src/net/packet_debug.cpp @@ -75,10 +75,10 @@ namespace net { // ignore the above, just hope we can write the full content of the packet print_len = pkt->size(); - fprintf(stderr, "Payload %i bytes from recorded layer begin (%p):\n", print_len, layer_begin); + fprintf(stderr, "Payload %i bytes from recorded layer begin (%p):", print_len, layer_begin); for(int i = 0; i < print_len; i++) { + if(i % 80 == 0) fprintf(stderr, "\n"); // break every 80th char fprintf(stderr, "%02x", *(layer_begin + i)); - if(i > 0 and i % 80 == 0) fprintf(stderr, "\n"); // break every 80th char } fprintf(stderr, "\n"); } From c994684cf7fed3e5304b9676e1ed5eb17cfcc1a1 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 3 Oct 2018 12:20:39 +0200 Subject: [PATCH 682/723] liveupdate: Optionally zero memory between kernel and heap end --- lib/LiveUpdate/hotswap64.asm | 15 ++++++++++++--- lib/LiveUpdate/update.cpp | 12 ++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/LiveUpdate/hotswap64.asm b/lib/LiveUpdate/hotswap64.asm index 1f001a4f6c..c5174deea7 100644 --- a/lib/LiveUpdate/hotswap64.asm +++ b/lib/LiveUpdate/hotswap64.asm @@ -29,11 +29,12 @@ ALIGN 16 ;; first six pointer arguments are passed in ;; RDI, RSI, RDX, RCX, R8, and R9 ;; hotswap64( -;; RDI: char* dest, +;; RDI: char* dest, ;; RSI: const char* base, ;; RDX: size_t len, -;; RCX: void* entry_function, -;; R8: void* reset_data) +;; RCX: void* entry_function, +;; R8: void* reset_data, +;; R9: void* zero_until) hotswap_amd64: ;; save soft reset data location and entry function mov rax, r8 @@ -47,6 +48,14 @@ hotswap_amd64: cld rep movsb + ;; memzero area between kernel and end of heap + cmp r9, 0 ;; ... but only if r9 != 0 + jz begin_enter_protected + mov rcx, r9 ;; rdi = kernel_end, r9 = zero_until + sub rcx, rdi ;; set rcx = zero_until - kernel_end + mov rax, 0 ;; memzero + rep stosb + begin_enter_protected: ; load 64-bit GDTR with 32-bit entries lgdt [gdtr64] diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index cb34bcdef4..9914ee2948 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -33,6 +33,7 @@ //#define LPRINT(x, ...) printf(x, ##__VA_ARGS__); #define LPRINT(x, ...) /** x **/ +#define LIU_ZERO_OLD_MEMORY static const int SECT_SIZE = 512; static const int ELF_MINIMUM = 164; @@ -41,7 +42,7 @@ extern "C" void solo5_exec(const char*, size_t); static void* HOTSWAP_AREA = (void*) 0x8000; extern "C" void hotswap(const char*, int, char*, uintptr_t, void*); extern "C" char __hotswap_length; -extern "C" void hotswap64(char*, const char*, int, uintptr_t, void*); +extern "C" void hotswap64(char*, const char*, int, uintptr_t, void*, void*); extern uint32_t hotswap64_len; extern void __x86_init_paging(void*); extern "C" void* __os_store_soft_reset(const void*, size_t); @@ -250,7 +251,14 @@ void LiveUpdate::exec(const buffer_t& blob, void* location) // copy hotswapping function to sweet spot memcpy(HOTSWAP_AREA, (void*) &hotswap64, hotswap64_len); /// the end - ((decltype(&hotswap64)) HOTSWAP_AREA)(phys_base, bin_data, bin_len, start_offset, sr_data); +#ifdef LIU_ZERO_OLD_MEMORY + ((decltype(&hotswap64)) HOTSWAP_AREA)(phys_base, bin_data, bin_len, + start_offset, /* binary entry point */ + sr_data, /* softreset location */ + (void*) OS::heap_end() /* zero memory until this location */); +#else + ((decltype(&hotswap64)) HOTSWAP_AREA)(phys_base, bin_data, bin_len, start_offset, sr_data, nullptr); +#endif # else # error "Unimplemented architecture" # endif From fdfaef73cb2df2ebd4ae1b91c20f426f78f1de1e Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 3 Oct 2018 13:38:39 +0200 Subject: [PATCH 683/723] dns: Use slightly safer C++ when parsing names --- src/net/dns/dns.cpp | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/net/dns/dns.cpp b/src/net/dns/dns.cpp index 5f9c6bf5a0..f2494c489a 100644 --- a/src/net/dns/dns.cpp +++ b/src/net/dns/dns.cpp @@ -325,60 +325,56 @@ namespace net std::string DNS::Request::rr_t::readName(const char* reader, const char* buffer, size_t tot_len, int& count) { std::string name(256, '\0'); - unsigned p = 0; - unsigned offset = 0; + unsigned namelen = 0; bool jumped = false; count = 1; - unsigned char* ureader = (unsigned char*) reader; + const auto* ureader = (unsigned char*) reader; while (*ureader) { if (*ureader >= 192) { // read 16-bit offset, mask out the 2 top bits - offset = ((*ureader) * 256 + *(ureader+1)) & 0x3FFF; // = 11000000 00000000 + uint16_t offset = (*ureader >> 8) | *(ureader+1); + offset &= 0x3fff; // remove 2 top bits if(UNLIKELY(offset > tot_len)) return {}; - - ureader = (unsigned char*) buffer + offset - 1; + + ureader = (unsigned char*) &buffer[offset]; jumped = true; // we have jumped to another location so counting wont go up! } else { - name[p++] = *ureader; + name[namelen++] = *ureader++; + // maximum label size + if(UNLIKELY(namelen > 63)) break; } - ureader++; // if we havent jumped to another location then we can count up if (jumped == false) count++; } - // maximum label size - if(UNLIKELY(p > 63)) - return {}; - - name.resize(p); + name.resize(namelen); // number of steps we actually moved forward in the packet if (jumped) count++; // now convert 3www6google3com0 to www.google.com - int len = p; // same as name.size() - int i; - for(i = 0; i < len; i++) + for(unsigned i = 0; i < name.size(); i++) { - p = name[i]; - for(unsigned j = 0; j < p; j++) + const uint8_t len = name[i]; + for(unsigned j = 0; j < len; j++) { name[i] = name[i+1]; i++; } name[i] = '.'; } - name[i - 1] = '\0'; // remove the last dot + // remove the last dot by resizing down + name.resize(name.size()-1); return name; } // readName() From 1eb248d28d33b887eeec309b878e1c5ac371cd41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Jerpetj=C3=B8n?= Date: Thu, 4 Oct 2018 10:17:15 +0200 Subject: [PATCH 684/723] FindPython2 for platforms with python3 as their main python --- cmake/post.service.cmake | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 73bf90329f..05617919db 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -14,6 +14,20 @@ message(STATUS "Target triple ${TRIPLE}") # defines $CAPABS depending on installation include(${CMAKE_CURRENT_LIST_DIR}/settings.cmake) +if (${CMAKE_VERSION} VERSION_LESS "3.12") + #ultimate fallback + set(Python2_EXECUTABLE python) + find_program(Python2 python2.7) + if (NOT Python2) + #brutal fallback + set(Python2_EXECUTABLE python) + else() + set(Python2_EXECUTABLE ${Python2}) + endif() +else() + find_package(Python2 COMPONENTS Interpreter) +endif() + # Arch-specific defines & options if ("${ARCH}" STREQUAL "x86_64") set(ARCH_INTERNAL "ARCH_X64") @@ -139,7 +153,7 @@ endif() if (EXISTS ${CMAKE_SOURCE_DIR}/nacl.txt) add_custom_command( OUTPUT nacl_content.cpp - COMMAND cat ${CMAKE_SOURCE_DIR}/nacl.txt | python ${INSTALL_LOC}/nacl/NaCl.py ${CMAKE_BINARY_DIR}/nacl_content.cpp + COMMAND cat ${CMAKE_SOURCE_DIR}/nacl.txt | ${Python2_EXECUTABLE} ${INSTALL_LOC}/nacl/NaCl.py ${CMAKE_BINARY_DIR}/nacl_content.cpp DEPENDS ${CMAKE_SOURCE_DIR}/nacl.txt ) add_library(nacl_content STATIC nacl_content.cpp) @@ -386,7 +400,7 @@ function(add_memdisk DISK) REALPATH BASE_DIR "${CMAKE_SOURCE_DIR}") add_custom_command( OUTPUT memdisk.o - COMMAND python ${INSTALL_LOC}/memdisk/memdisk.py --file memdisk.asm ${DISK_RELPATH} + COMMAND ${Python2_EXECUTABLE} ${INSTALL_LOC}/memdisk/memdisk.py --file memdisk.asm ${DISK_RELPATH} COMMAND nasm -f ${CMAKE_ASM_NASM_OBJECT_FORMAT} memdisk.asm -o memdisk.o DEPENDS ${DISK_RELPATH} ) From 674165975f7cff14008dfd50fec77338e7bd3b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Jerpetj=C3=B8n?= Date: Thu, 4 Oct 2018 10:22:23 +0200 Subject: [PATCH 685/723] removed extra entry --- cmake/post.service.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmake/post.service.cmake b/cmake/post.service.cmake index 05617919db..28387b02b8 100644 --- a/cmake/post.service.cmake +++ b/cmake/post.service.cmake @@ -15,8 +15,6 @@ message(STATUS "Target triple ${TRIPLE}") include(${CMAKE_CURRENT_LIST_DIR}/settings.cmake) if (${CMAKE_VERSION} VERSION_LESS "3.12") - #ultimate fallback - set(Python2_EXECUTABLE python) find_program(Python2 python2.7) if (NOT Python2) #brutal fallback From 74c74330bdbaf9280a9fbfd2ef85071a3f68158d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Jerpetj=C3=B8n?= Date: Thu, 4 Oct 2018 10:58:05 +0200 Subject: [PATCH 686/723] api: added read and write memory barrier functions --- api/arch.hpp | 5 +++-- api/arch/x86_64.hpp | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/api/arch.hpp b/api/arch.hpp index 0bf8af3cb3..3fb0c0c83c 100644 --- a/api/arch.hpp +++ b/api/arch.hpp @@ -1,4 +1,4 @@ -// -*-C++-*- +// -*-C++-*- // This file is a part of the IncludeOS unikernel - www.includeos.org // // Copyright 2017 Oslo and Akershus University College of Applied Sciences @@ -36,7 +36,8 @@ extern void __arch_install_irq(uint8_t, void(*)()); extern void __arch_subscribe_irq(uint8_t); extern void __arch_unsubscribe_irq(uint8_t); extern void __arch_preempt_forever(void(*)()); - +extern inline void __arch_read_memory_barrier() noexcept; +extern inline void __arch_write_memory_barrier() noexcept; inline void __arch_hw_barrier() noexcept; inline void __sw_barrier() noexcept; diff --git a/api/arch/x86_64.hpp b/api/arch/x86_64.hpp index 75e7958c91..0e04202a58 100644 --- a/api/arch/x86_64.hpp +++ b/api/arch/x86_64.hpp @@ -1,4 +1,4 @@ -// -*-C++-*- +// -*-C++-*- // This file is a part of the IncludeOS unikernel - www.includeos.org // // Copyright 2017 Oslo and Akershus University College of Applied Sciences @@ -21,6 +21,13 @@ #define ARCH_x86 +inline void __arch_read_memory_barrier() noexcept { + __asm volatile("lfence" ::: "memory"); +} +inline void __arch_write_memory_barrier() noexcept { + __asm volatile("mfence" ::: "memory"); +} + inline uint64_t __arch_cpu_cycles() noexcept { uint32_t hi, lo; asm("rdtsc" : "=a"(lo), "=d"(hi)); From a3531704d7f5cdac55589f2c3b488e53b39237f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Jerpetj=C3=B8n?= Date: Thu, 4 Oct 2018 11:05:15 +0200 Subject: [PATCH 687/723] drivers: added read barrier to prevent speculative read into descriptor before generation is set by sender --- src/drivers/vmxnet3.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index da1c4baafd..e03df2d18b 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -506,12 +506,17 @@ bool vmxnet3::receive_handler(const int Q) auto& comp = dma->rx[Q].comp[idx]; // break when exiting this generation if (gen != (comp.flags & VMXNET3_RXCF_GEN)) break; + + /* prevent speculative pre read ahead of comp content*/ + __arch_read_memory_barrier(); + rx[Q].consumers++; rx[Q].prod_count--; int desc = comp.index % vmxnet3::NUM_RX_DESC; // mask out length int len = comp.len & (VMXNET3_MAX_BUFFER_LEN-1); + // get buffer and construct packet assert(rx[Q].buffers[desc] != nullptr); recvq.push_back(recv_packet(rx[Q].buffers[desc], len)); From a902f5582d7e1688a16b64060591a491d01409a8 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 2 Oct 2018 19:17:19 +0200 Subject: [PATCH 688/723] ip4: Decrement TTL for loopback packets --- src/net/ip4/ip4.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/net/ip4/ip4.cpp b/src/net/ip4/ip4.cpp index 898dbb3039..389c35bc36 100644 --- a/src/net/ip4/ip4.cpp +++ b/src/net/ip4/ip4.cpp @@ -269,7 +269,12 @@ namespace net { // Send loopback packets right back if (UNLIKELY(stack_.is_valid_source(packet->ip_dst()))) { - PRINT(" Destination address is loopback \n"); + PRINT(" Loopback packet returned SRC %s DST %s\n", + packet->ip_src().to_string().c_str(), + packet->ip_dst().to_string().c_str() + ); + // to avoid loops, lets decrement hop count here + packet->decrement_ttl(); IP4::receive(std::move(packet), false); return; } From e8b075c891657965460fe90e8924fbcc65ad7841 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 2 Oct 2018 19:17:58 +0200 Subject: [PATCH 689/723] inet: Disallow loopback packets on the way in --- api/net/inet.hpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/api/net/inet.hpp b/api/net/inet.hpp index 2a436935e8..295d51d50d 100644 --- a/api/net/inet.hpp +++ b/api/net/inet.hpp @@ -400,7 +400,6 @@ namespace net { IP4::addr get_source_addr(IP4::addr dest) { - if (dest.is_loopback()) return {127,0,0,1}; @@ -412,7 +411,6 @@ namespace net { IP6::addr get_source_addr(IP6::addr dest) { - if (dest.is_loopback()) return ip6::Addr{0,0,0,1}; @@ -426,10 +424,10 @@ namespace net { { return addr.is_v4() ? is_valid_source4(addr.v4()) : is_valid_source6(addr.v6()); } bool is_valid_source4(IP4::addr src) const - { return src == ip_addr() or is_loopback(src); } + { return src == ip_addr(); } bool is_valid_source6(const IP6::addr& src) const - { return src == ip6_addr() or is_loopback(src) or src.is_multicast(); } + { return src == ip6_addr() or src.is_multicast(); } std::shared_ptr& conntrack() { return conntrack_; } From ef9b5fd1744b5fbe2b6f006e3d499cd8548dd9b3 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Tue, 2 Oct 2018 19:22:59 +0200 Subject: [PATCH 690/723] inet: Ignore some invalid IP4 packets in error reporting --- src/net/inet.cpp | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/src/net/inet.cpp b/src/net/inet.cpp index 5044c935bb..4d6504ef62 100644 --- a/src/net/inet.cpp +++ b/src/net/inet.cpp @@ -130,28 +130,34 @@ Inet::Inet(hw::Nic& nic) } void Inet::error_report(Error& err, Packet_ptr orig_pckt) { - auto pckt_ip4 = static_unique_ptr_cast(std::move(orig_pckt)); - bool too_big = false; + // if its a forged packet, it might be too small + if (orig_pckt->size() < 40) return; + auto pckt_ip4 = static_unique_ptr_cast(std::move(orig_pckt)); // Get the destination to the original packet - Socket dest = [](std::unique_ptr& pkt)->Socket { - switch (pkt->ip_protocol()) { - case Protocol::UDP: { - const auto& udp = static_cast(*pkt); - return udp.destination(); - } - case Protocol::TCP: { - auto tcp = tcp::Packet4_view(std::move(pkt)); - auto dst = tcp.destination(); - pkt = static_unique_ptr_cast(tcp.release()); - return dst; + const Socket dest = + [] (std::unique_ptr& pkt)->Socket + { + // if its a forged packet, it might not be IPv4 + if (pkt->is_ipv4() == false) return {}; + // switch on IP4 protocol + switch (pkt->ip_protocol()) { + case Protocol::UDP: { + const auto& udp = static_cast(*pkt); + return udp.destination(); + } + case Protocol::TCP: { + auto tcp = tcp::Packet4_view(std::move(pkt)); + auto dst = tcp.destination(); + pkt = static_unique_ptr_cast(tcp.release()); + return dst; + } + default: + return {}; } - default: - return {}; - } - }(pckt_ip4); - + }(pckt_ip4); + bool too_big = false; if (err.is_icmp()) { auto* icmp_err = dynamic_cast(&err); if (icmp_err == nullptr) { From 35622d7fc55710ceac03755d3a5a84146b28661f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 3 Oct 2018 13:38:39 +0200 Subject: [PATCH 691/723] dns: Use slightly safer C++ when parsing names --- src/net/dns/dns.cpp | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/src/net/dns/dns.cpp b/src/net/dns/dns.cpp index 5f9c6bf5a0..f2494c489a 100644 --- a/src/net/dns/dns.cpp +++ b/src/net/dns/dns.cpp @@ -325,60 +325,56 @@ namespace net std::string DNS::Request::rr_t::readName(const char* reader, const char* buffer, size_t tot_len, int& count) { std::string name(256, '\0'); - unsigned p = 0; - unsigned offset = 0; + unsigned namelen = 0; bool jumped = false; count = 1; - unsigned char* ureader = (unsigned char*) reader; + const auto* ureader = (unsigned char*) reader; while (*ureader) { if (*ureader >= 192) { // read 16-bit offset, mask out the 2 top bits - offset = ((*ureader) * 256 + *(ureader+1)) & 0x3FFF; // = 11000000 00000000 + uint16_t offset = (*ureader >> 8) | *(ureader+1); + offset &= 0x3fff; // remove 2 top bits if(UNLIKELY(offset > tot_len)) return {}; - - ureader = (unsigned char*) buffer + offset - 1; + + ureader = (unsigned char*) &buffer[offset]; jumped = true; // we have jumped to another location so counting wont go up! } else { - name[p++] = *ureader; + name[namelen++] = *ureader++; + // maximum label size + if(UNLIKELY(namelen > 63)) break; } - ureader++; // if we havent jumped to another location then we can count up if (jumped == false) count++; } - // maximum label size - if(UNLIKELY(p > 63)) - return {}; - - name.resize(p); + name.resize(namelen); // number of steps we actually moved forward in the packet if (jumped) count++; // now convert 3www6google3com0 to www.google.com - int len = p; // same as name.size() - int i; - for(i = 0; i < len; i++) + for(unsigned i = 0; i < name.size(); i++) { - p = name[i]; - for(unsigned j = 0; j < p; j++) + const uint8_t len = name[i]; + for(unsigned j = 0; j < len; j++) { name[i] = name[i+1]; i++; } name[i] = '.'; } - name[i - 1] = '\0'; // remove the last dot + // remove the last dot by resizing down + name.resize(name.size()-1); return name; } // readName() From e7f1b718d497a9df7e9484a107d0b4e807acd45a Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 4 Oct 2018 11:31:40 +0200 Subject: [PATCH 692/723] udp: Validate UDP packet length --- api/net/ip4/packet_udp.hpp | 12 +++++++++++- src/net/ip4/udp.cpp | 6 ++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/api/net/ip4/packet_udp.hpp b/api/net/ip4/packet_udp.hpp index 61db4defc9..30319248b0 100644 --- a/api/net/ip4/packet_udp.hpp +++ b/api/net/ip4/packet_udp.hpp @@ -77,7 +77,10 @@ namespace net } uint16_t data_length() const noexcept { - return length() - sizeof(UDP::header); + const uint16_t hdr_len = ip_header_length(); + uint16_t real_length = size() - hdr_len; + uint16_t final_length = std::min(real_length, length()); + return final_length - sizeof(UDP::header); } Byte* data() @@ -122,6 +125,13 @@ namespace net return total; } + bool validate_length() const noexcept { + uint16_t hdr_len = ip_header_length(); + if ((unsigned) size() < hdr_len + sizeof(UDP::header)) return false; + if (length() < sizeof(UDP::header)) return false; + return true; + } + private: // Sets the correct length for UDP and the packet void set_length(uint16_t newlen) diff --git a/src/net/ip4/udp.cpp b/src/net/ip4/udp.cpp index fc74c50c46..d79578597a 100644 --- a/src/net/ip4/udp.cpp +++ b/src/net/ip4/udp.cpp @@ -43,6 +43,12 @@ namespace net { { auto udp_packet = static_unique_ptr_cast(std::move(pckt)); + if (udp_packet->validate_length() == false) { + PRINT("<%s> UDP: Invalid packet received (too short). Drop!\n", + stack_.ifname().c_str()); + return; + } + PRINT("<%s> UDP", stack_.ifname().c_str()); PRINT("\t Source port: %u, Dest. Port: %u Length: %u\n", From b7bc2e8799eb407b28bbce6051fed01a65b4218f Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 4 Oct 2018 11:39:37 +0200 Subject: [PATCH 693/723] dns: Validate and terminate on reading errors --- src/net/dns/client.cpp | 6 +++++- src/net/dns/dns.cpp | 38 +++++++++++++++++++++++++------------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/net/dns/client.cpp b/src/net/dns/client.cpp index d0603029ba..dfbeccd263 100644 --- a/src/net/dns/client.cpp +++ b/src/net/dns/client.cpp @@ -20,7 +20,11 @@ namespace net { +#ifdef LIBFUZZER_ENABLED + Timer::duration_t DNSClient::DEFAULT_RESOLVE_TIMEOUT{std::chrono::seconds(9999)}; +#else Timer::duration_t DNSClient::DEFAULT_RESOLVE_TIMEOUT{std::chrono::seconds(5)}; +#endif Timer::duration_t DNSClient::DEFAULT_FLUSH_INTERVAL{std::chrono::seconds(30)}; std::chrono::seconds DNSClient::DEFAULT_CACHE_TTL{std::chrono::seconds(60)}; @@ -134,7 +138,7 @@ namespace net } else { - debug(" Cannot find matching DNS Request with transid=%u\n", ntohs(reply.id)); + debug(" Cannot find matching DNS Request with transid=%u\n", ntohs(reply.id)); } } diff --git a/src/net/dns/dns.cpp b/src/net/dns/dns.cpp index f2494c489a..d66c3fbd37 100644 --- a/src/net/dns/dns.cpp +++ b/src/net/dns/dns.cpp @@ -194,18 +194,23 @@ namespace net if(UNLIKELY(reader > (buffer + len))) return false; - // parse answers - for(int i = 0; i < ntohs(dns->ans_count); i++) - answers.emplace_back(reader, buffer, len); - - // parse authorities - for (int i = 0; i < ntohs(dns->auth_count); i++) - auth.emplace_back(reader, buffer, len); - - // parse additional - for (int i = 0; i < ntohs(dns->add_count); i++) - addit.emplace_back(reader, buffer, len); - + try { + // parse answers + for(int i = 0; i < ntohs(dns->ans_count); i++) + answers.emplace_back(reader, buffer, len); + + // parse authorities + for (int i = 0; i < ntohs(dns->auth_count); i++) + auth.emplace_back(reader, buffer, len); + + // parse additional + for (int i = 0; i < ntohs(dns->add_count); i++) + addit.emplace_back(reader, buffer, len); + } + catch (const std::runtime_error&) { + // packet probably too short + } + return true; } @@ -259,10 +264,16 @@ namespace net DNS::Request::rr_t::rr_t(const char*& reader, const char* buffer, size_t len) { int stop; - this->name = readName(reader, buffer, len, stop); + assert(stop >= 0); reader += stop; + int remaining = len - (reader - buffer); + assert(remaining <= (int) len); + // invalid request if there is no room for resources + if (remaining < (int) sizeof(rr_data)) + throw std::runtime_error("Nothing left to parse"); + // extract resource data header this->resource = *(rr_data*) reader; reader += sizeof(rr_data); @@ -357,6 +368,7 @@ namespace net } name.resize(namelen); + if (name.empty()) return name; // number of steps we actually moved forward in the packet if (jumped) From 2865b7c14e72494119adfd71348c1f88157ac553 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Thu, 4 Oct 2018 11:55:31 +0200 Subject: [PATCH 694/723] Dockerfile: Added labels, printing version when building service --- Dockerfile | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 922813efd0..ffd9425026 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,4 @@ FROM ubuntu:xenial as base -LABEL maintainer="Martin Nordsletten, S Taiyeba Haroon" RUN apt-get update && apt-get -y install \ sudo \ @@ -26,10 +25,16 @@ RUN USER=docker && \ mkdir -p /etc/fixuid && \ printf "user: $USER\ngroup: $GROUP\npaths:\n - /service\n" > /etc/fixuid/config.yml +ARG CUSTOM_TAG +LABEL org.label-schema.schema-version="1.0" \ + org.label-schema.name="IncludeOS builder" \ + org.label-schema.vendor="IncludeOS" \ + org.label-schema.version=$CUSTOM_TAG \ + org.label-schema.vcs-url="https://github.com/hioa-cs/includeos" + RUN echo "LANG=C.UTF-8" > /etc/default/locale ######################### - FROM base as source-build RUN apt-get update && apt-get -y install \ @@ -43,7 +48,7 @@ RUN mkdir -p /root/IncludeOS WORKDIR /root/IncludeOS COPY . . -# Ability to specify custom tag that overwrites any existing tag. This will then match the reported IncludeOS version +# Ability to specify custom tag that overwrites any existing tag. This will then match what IncludeOS reports itself. ARG CUSTOM_TAG RUN git describe --tags --dirty > /ios_version.txt RUN : ${CUSTOM_TAG:=$(git describe --tags)} && git tag -d $(git describe --tags); git tag $CUSTOM_TAG && git describe --tags --dirty > /custom_tag.txt @@ -52,7 +57,6 @@ RUN : ${CUSTOM_TAG:=$(git describe --tags)} && git tag -d $(git describe --tags) RUN ./install.sh -n ############################# - FROM base as grubify RUN apt-get update && apt-get -y install \ @@ -64,7 +68,6 @@ COPY --from=source-build /usr/local/includeos/scripts/grubify.sh /home/ubuntu/In ENTRYPOINT ["fixuid", "/home/ubuntu/IncludeOS_install/includeos/scripts/grubify.sh"] ########################### - FROM base as build RUN apt-get update && apt-get -y install \ @@ -78,6 +81,11 @@ RUN apt-get update && apt-get -y install \ apt-get remove -y python-pip && \ apt autoremove -y +ARG VCS_REF="Check file /ios_version.txt inside container" +LABEL org.label-schema.description="Build a service using IncludeOS" \ + org.label-schema.vcs-ref=$VCS_REF \ + org.label-schema.docker.cmd="docker run -v $PWD:/service " + WORKDIR /service COPY --from=source-build /usr/local/includeos /usr/local/includeos/ @@ -93,5 +101,6 @@ ENTRYPOINT ["/entrypoint.sh"] CMD mkdir -p build && \ cd build && \ cp $(find /usr/local/includeos -name chainloader) /service/build/chainloader && \ + echo "IncludeOS version:" $(cat /ios_version.txt) "tag:" $(cat /custom_tag.txt) && \ cmake .. && \ make From cad3d06687ec1c281a27d7d8d2375972cddbb5b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Thu, 4 Oct 2018 11:55:37 +0200 Subject: [PATCH 695/723] net: Include offset bytes (since size() is calculated from layer) in packet debug --- src/net/packet_debug.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/net/packet_debug.cpp b/src/net/packet_debug.cpp index b6e095cc79..50209c7d9d 100644 --- a/src/net/packet_debug.cpp +++ b/src/net/packet_debug.cpp @@ -38,8 +38,9 @@ namespace net { fprintf(stderr, "*** Last packet:\n"); fprintf(stderr, "Buffer: Begin: %p End: %p Size: %i\n", pkt->buf(), pkt->buffer_end(), pkt->bufsize()); + const size_t offset = pkt->layer_begin() - layer_begin; fprintf(stderr, "Layer: Recorded: %p Current: %p (%lub offset)\n", - layer_begin, pkt->layer_begin(), pkt->layer_begin() - layer_begin); + layer_begin, pkt->layer_begin(), offset); fprintf(stderr, "Size: %i ", pkt->size()); fprintf(stderr, "Capacity: %i ", pkt->capacity()); fprintf(stderr, "Data end: %p \n", pkt->data_end()); @@ -73,7 +74,7 @@ namespace net { } // ignore the above, just hope we can write the full content of the packet - print_len = pkt->size(); + print_len = offset + pkt->size(); fprintf(stderr, "Payload %i bytes from recorded layer begin (%p):", print_len, layer_begin); for(int i = 0; i < print_len; i++) { From 6717b979a206a60475b13c260514a6a8691187ee Mon Sep 17 00:00:00 2001 From: Ingve Vormestrand Date: Thu, 4 Oct 2018 13:05:18 +0200 Subject: [PATCH 696/723] readme: minor updates --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 927e2df2c0..94990cffc2 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,11 @@ IncludeOS is free software, with "no warranties or restrictions of any kind". * **KVM, VirtualBox and VMWare support** with full virtualization, using [x86 hardware virtualization](https://en.wikipedia.org/wiki/X86_virtualization), available on most modern x86 CPUs. IncludeOS will run on any x86 hardware platform, even on a physical x86 computer, given appropriate drivers. Officially, we develop for- and test on [Linux KVM](http://www.linux-kvm.org/page/Main_Page), and VMWare [ESXi](https://www.vmware.com/products/esxi-and-esx.html)/[Fusion](https://www.vmware.com/products/fusion.html) which means that you can run your IncludeOS service on Linux, Microsoft Windows and macOS, as well as on cloud providers such as [Google Compute Engine](http://www.includeos.org/blog/2017/includeos-on-google-compute-engine.html), [OpenStack](https://www.openstack.org/) and VMWare [vcloud](https://www.vmware.com/products/vcloud-suite.html). * **Instant boot:** IncludeOS on Qemu/kvm boots in about 300ms but IBM Research has also integrated IncludeOS with [Solo5/uKVM](https://github.com/Solo5/solo5), providing boot times as low as 10 milliseconds. * **C++11/14 support** - * Full C++11/14 language support with [clang](http://clang.llvm.org) v3.8 and later. + * Full C++11/14/17 language support with [clang](http://clang.llvm.org) 5 and later. * Standard C++ library (STL) [libc++](http://libcxx.llvm.org) from [LLVM](http://llvm.org/). * Exceptions and stack unwinding (currently using [libgcc](https://gcc.gnu.org/onlinedocs/gccint/Libgcc.html)). * *Note:* Certain language features, such as threads and filestreams are currently missing backend support. -* **Standard C library** using [newlib](https://sourceware.org/newlib/) from [Red Hat](http://www.redhat.com/). +* **Standard C library** using [musl libc](http://www.musl-libc.org/). * **Virtio and vmxnet3 Network drivers** with DMA. [Virtio](https://www.oasis-open.org/committees/tc_home.php?wg_abbrev=virtio) provides a highly efficient and widely supported I/O virtualization. vmxnet3 is the VMWare equivalent. * **A highly modular TCP/IP-stack**. @@ -42,11 +42,11 @@ A longer list of features and limitations can be found on our [documentation sit ### Set custom location and compiler -By default the project is installed to /usr/local/includeos. +By default the project is installed to `/usr/local/includeos`. -However, it is recommended to choose a custom location as well as select the compiler we want clang to find. In this document we assume you install IncludeOS in your home directory, in the folder ~/includeos. +However, it is recommended to choose a custom location as well as select the compiler we want clang to find. In this document we assume you install IncludeOS in your home directory, in the folder `~/includeos`. -To do this we can edit ~/.bash_profile (mac os) or ~/.bashrc (linux), adding these lines at the end of the file: +To do this we can edit `~/.bash_profile` (mac os) or `~/.bashrc` (linux), adding these lines at the end of the file: ``` export INCLUDEOS_PREFIX=~/includeos/ From 1486298855468bcc390f5ed946c8eca93c4da41c Mon Sep 17 00:00:00 2001 From: Ingve Vormestrand Date: Thu, 4 Oct 2018 13:08:48 +0200 Subject: [PATCH 697/723] readme: minor copyedit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 94990cffc2..1b2d515fbb 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ IncludeOS is free software, with "no warranties or restrictions of any kind". * **Extreme memory footprint**: A minimal bootable 64-bit web server, including operating system components and a anything needed from the C/C++ standard libraries is currently 2.5 MB. * **KVM, VirtualBox and VMWare support** with full virtualization, using [x86 hardware virtualization](https://en.wikipedia.org/wiki/X86_virtualization), available on most modern x86 CPUs. IncludeOS will run on any x86 hardware platform, even on a physical x86 computer, given appropriate drivers. Officially, we develop for- and test on [Linux KVM](http://www.linux-kvm.org/page/Main_Page), and VMWare [ESXi](https://www.vmware.com/products/esxi-and-esx.html)/[Fusion](https://www.vmware.com/products/fusion.html) which means that you can run your IncludeOS service on Linux, Microsoft Windows and macOS, as well as on cloud providers such as [Google Compute Engine](http://www.includeos.org/blog/2017/includeos-on-google-compute-engine.html), [OpenStack](https://www.openstack.org/) and VMWare [vcloud](https://www.vmware.com/products/vcloud-suite.html). * **Instant boot:** IncludeOS on Qemu/kvm boots in about 300ms but IBM Research has also integrated IncludeOS with [Solo5/uKVM](https://github.com/Solo5/solo5), providing boot times as low as 10 milliseconds. -* **C++11/14 support** +* **Modern C++ support** * Full C++11/14/17 language support with [clang](http://clang.llvm.org) 5 and later. * Standard C++ library (STL) [libc++](http://libcxx.llvm.org) from [LLVM](http://llvm.org/). * Exceptions and stack unwinding (currently using [libgcc](https://gcc.gnu.org/onlinedocs/gccint/Libgcc.html)). From 5f087b08a77ecd2bebfaf5753257e201d5e1c17b Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 4 Oct 2018 14:56:48 +0200 Subject: [PATCH 698/723] ip4: Validate IP header length, TCP and UDP header room --- api/net/ip4/ip4.hpp | 4 ++-- api/net/ip4/packet_ip4.hpp | 4 ++++ api/net/ip4/packet_udp.hpp | 5 +---- api/net/tcp/packet_view.hpp | 4 ++++ src/net/ip4/ip4.cpp | 7 +++++-- src/net/tcp/listener.cpp | 23 ++++++++++++++++------- src/net/tcp/tcp.cpp | 6 ++++-- 7 files changed, 36 insertions(+), 17 deletions(-) diff --git a/api/net/ip4/ip4.hpp b/api/net/ip4/ip4.hpp index 625df96702..936cfcd1f7 100644 --- a/api/net/ip4/ip4.hpp +++ b/api/net/ip4/ip4.hpp @@ -39,8 +39,8 @@ namespace net { public: enum class Drop_reason - { None, Bad_source, Bad_destination, Wrong_version, Wrong_checksum, - Unknown_proto, TTL0 }; + { None, Bad_source, Bad_length, Bad_destination, + Wrong_version, Wrong_checksum, Unknown_proto, TTL0 }; enum class Direction { Upstream, Downstream }; diff --git a/api/net/ip4/packet_ip4.hpp b/api/net/ip4/packet_ip4.hpp index 4352285f2a..cf6f7593bb 100644 --- a/api/net/ip4/packet_ip4.hpp +++ b/api/net/ip4/packet_ip4.hpp @@ -246,6 +246,10 @@ namespace net { return {ip_data_ptr(), ip_data_length()}; } + bool validate_length() const noexcept { + return this->size() == ip_header_length() + ip_data_length(); + } + protected: /** Get pointer to IP data */ diff --git a/api/net/ip4/packet_udp.hpp b/api/net/ip4/packet_udp.hpp index 30319248b0..7ede23238a 100644 --- a/api/net/ip4/packet_udp.hpp +++ b/api/net/ip4/packet_udp.hpp @@ -126,10 +126,7 @@ namespace net } bool validate_length() const noexcept { - uint16_t hdr_len = ip_header_length(); - if ((unsigned) size() < hdr_len + sizeof(UDP::header)) return false; - if (length() < sizeof(UDP::header)) return false; - return true; + return length() >= sizeof(UDP::header); } private: diff --git a/api/net/tcp/packet_view.hpp b/api/net/tcp/packet_view.hpp index 42810452ba..2f4a707c3c 100644 --- a/api/net/tcp/packet_view.hpp +++ b/api/net/tcp/packet_view.hpp @@ -182,6 +182,10 @@ class Packet_v { inline size_t fill(const uint8_t* buffer, size_t length); + bool validate_length() const noexcept { + return ip_data_length() >= tcp_header_length(); + } + // Util // seq_t end() const noexcept diff --git a/src/net/ip4/ip4.cpp b/src/net/ip4/ip4.cpp index 389c35bc36..cedef595d0 100644 --- a/src/net/ip4/ip4.cpp +++ b/src/net/ip4/ip4.cpp @@ -59,12 +59,15 @@ namespace net { IP4::IP_packet_ptr IP4::drop_invalid_in(IP4::IP_packet_ptr packet) { - IP4::Direction up = IP4::Direction::Upstream; - + const IP4::Direction up = IP4::Direction::Upstream; // RFC-1122 3.2.1.1, Silently discard Version != 4 if (UNLIKELY(not packet->is_ipv4())) return drop(std::move(packet), up, Drop_reason::Wrong_version); + // Don't read from data before we know the length is sane + if (UNLIKELY(not packet->validate_length())) + return drop(std::move(packet), up, Drop_reason::Bad_length); + // RFC-1122 3.2.1.2, Verify IP checksum, silently discard bad dgram if (UNLIKELY(packet->compute_ip_checksum() != 0)) return drop(std::move(packet), up, Drop_reason::Wrong_checksum); diff --git a/src/net/tcp/listener.cpp b/src/net/tcp/listener.cpp index dad0f7fc0b..c2d4940886 100644 --- a/src/net/tcp/listener.cpp +++ b/src/net/tcp/listener.cpp @@ -22,6 +22,15 @@ using namespace net; using namespace tcp; +#define VERBOSE_TCP_LISTENER +#ifdef VERBOSE_TCP_LISTENER +#define TCPL_PRINT1(x, ...) printf(x, ##__VA_ARGS__) +#define TCPL_PRINT2(x, ...) printf(x, ##__VA_ARGS__) +#else +#define TCPL_PRINT1(x, ...) /* x */ +#define TCPL_PRINT2(x, ...) /* x */ +#endif + Listener::Listener(TCP& host, Socket local, ConnectCallback cb, const bool ipv6_only) : host_(host), local_(local), syn_queue_(), on_accept_({this, &Listener::default_on_accept}), @@ -40,7 +49,7 @@ bool Listener::syn_queue_full() const void Listener::segment_arrived(Packet_view& packet) { - debug2(" Received packet: %s\n", + TCPL_PRINT2(" Received packet: %s\n", packet.to_string().c_str()); auto it = std::find_if(syn_queue_.begin(), syn_queue_.end(), @@ -55,7 +64,7 @@ void Listener::segment_arrived(Packet_view& packet) { debug(" Found packet receiver: %s\n", conn->to_string().c_str()); conn->segment_arrived(packet); - debug2(" Connection done handling segment\n"); + TCPL_PRINT2(" Connection done handling segment\n"); return; } // if it's a new attempt (SYN) @@ -76,11 +85,11 @@ void Listener::segment_arrived(Packet_view& packet) { return; // remove oldest connection if queue is full - debug2(" SynQueue: %u\n", syn_queue_.size()); + TCPL_PRINT2(" SynQueue: %u\n", syn_queue_.size()); // SYN queue is full if(syn_queue_.size() >= host_.max_syn_backlog()) { - debug2(" Queue is full\n"); + TCPL_PRINT2(" Queue is full\n"); Expects(not syn_queue_.empty()); debug(" Connection %s dropped to make room for new connection\n", syn_queue_.back()->to_string().c_str()); @@ -100,14 +109,14 @@ void Listener::segment_arrived(Packet_view& packet) { debug(" Connection %s created\n", conn->to_string().c_str()); conn->segment_arrived(packet); - debug2(" Connection done handling segment\n"); + TCPL_PRINT2(" Connection done handling segment\n"); return; } - debug2(" No receipent\n"); + TCPL_PRINT2(" No receipent\n"); } void Listener::remove(Connection_ptr conn) { - debug2(" Try remove %s\n", conn->to_string().c_str()); + TCPL_PRINT2(" Try remove %s\n", conn->to_string().c_str()); auto it = syn_queue_.begin(); while(it != syn_queue_.end()) { diff --git a/src/net/tcp/tcp.cpp b/src/net/tcp/tcp.cpp index e0f8fddf64..fb1ed06be5 100644 --- a/src/net/tcp/tcp.cpp +++ b/src/net/tcp/tcp.cpp @@ -212,13 +212,15 @@ void TCP::receive(Packet_view& packet) (*packets_rx_)++; assert(get_cpuid() == SMP::cpu_id()); - //auto packet = static_unique_ptr_cast(std::move(packet_ptr)); - // validate some unlikely but invalid packet properties if (UNLIKELY(packet.src_port() == 0)) { drop(packet); return; } + if (UNLIKELY(packet.validate_length() == false)) { + drop(packet); + return; + } // Validate checksum if (UNLIKELY(packet.compute_tcp_checksum() != 0)) { From 812924ffdee581d15e06d2beb2d3be55ccf1be79 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 4 Oct 2018 14:57:07 +0200 Subject: [PATCH 699/723] dns: Verify enough room to start parsing name --- src/net/dns/dns.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/net/dns/dns.cpp b/src/net/dns/dns.cpp index d66c3fbd37..e91b3a1f0c 100644 --- a/src/net/dns/dns.cpp +++ b/src/net/dns/dns.cpp @@ -263,6 +263,9 @@ namespace net DNS::Request::rr_t::rr_t(const char*& reader, const char* buffer, size_t len) { + // don't call readName if we are already out of buffer + if (reader >= buffer + len) + throw std::runtime_error("Nothing left to parse"); int stop; this->name = readName(reader, buffer, len, stop); assert(stop >= 0); From efa15d0e69252e76e4f644377b00c14d0cfa4c66 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 4 Oct 2018 16:37:13 +0200 Subject: [PATCH 700/723] test: Send mega-sized datagram to IncludeOS and back --- test/net/integration/udp/CMakeLists.txt | 3 +-- test/net/integration/udp/service.cpp | 32 ++++++++++++------------- test/net/integration/udp/test.py | 22 +++++++++++++++-- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/test/net/integration/udp/CMakeLists.txt b/test/net/integration/udp/CMakeLists.txt index 3ee122c4b3..677412c6b4 100644 --- a/test/net/integration/udp/CMakeLists.txt +++ b/test/net/integration/udp/CMakeLists.txt @@ -11,9 +11,8 @@ project (test_udp) set(SERVICE_NAME "UDP test service") set(BINARY "test_udp") -set(MAX_MEM 128) set(SOURCES service.cpp) -set(DRIVERS virtionet) +set(DRIVERS virtionet ip4_reassembly) # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/net/integration/udp/service.cpp b/test/net/integration/udp/service.cpp index 4ee1ce913c..c4a00df8ff 100644 --- a/test/net/integration/udp/service.cpp +++ b/test/net/integration/udp/service.cpp @@ -17,8 +17,6 @@ #include #include -#include - using namespace net; void Service::start() @@ -27,10 +25,7 @@ void Service::start() inet.network_config({ 10, 0, 0, 55 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 1 } ); // Gateway - printf("Service IP address is %s\n", inet.ip_addr().to_string().c_str()); - - // UDP - const UDP::port_t port = 4242; + const uint16_t port = 4242; auto& sock = inet.udp().bind(port); sock.on_read( @@ -38,17 +33,20 @@ void Service::start() UDP::addr_t addr, UDP::port_t port, const char* data, size_t len) { - std::string strdata(data, len); - CHECK(1, "Getting UDP data from %s: %d -> %s", - addr.to_string().c_str(), port, strdata.c_str()); - // send the same thing right back! - using namespace std::chrono; - Timers::oneshot(100ms, - Timers::handler_t::make_packed( - [&sock, addr, port, data, len](Timers::id_t) { - sock.sendto(addr, port, data, len); - })); - }); + CHECK(1, "UDP data from %s:%u -> %.*s", + addr.to_string().c_str(), port, std::min((int) len, 16), data); + // send the same thing right back! + printf("Got %lu bytes\n", len); + while (len >= 1472) + { + sock.sendto(addr, port, data, 1472); + data += 1472; + len -= 1472; + } + if (len > 0) { + sock.sendto(addr, port, data, len); + } + }); INFO("UDP test", "Listening on port %d\n", port); } diff --git a/test/net/integration/udp/test.py b/test/net/integration/udp/test.py index d7efddea51..89ae738982 100755 --- a/test/net/integration/udp/test.py +++ b/test/net/integration/udp/test.py @@ -40,8 +40,26 @@ def UDP_test(trigger_line): print " Sent: {}".format(data) print " Received: {}".format(received) - if received == data: - vmrunner.vms[0].exit(0, "Test completed without errors") + if received != data: return False + + data = "x" * 1472 + sock.sendto(data, (HOST, PORT)) + received = sock.recv(1500) + if received != data: + print " Did not receive long string: {}".format(received) + return False + + data = "x" * 32000 + sock.sendto(data, (HOST, PORT)) + received = bytearray() + while (len(received) < len(data)): + received.extend(sock.recv(len(data))) + print "RECEIVED: ", len(received) + if received != data: + print " Did not receive mega string (64k)" + return False + + vmrunner.vms[0].exit(0, "Test completed without errors") # Add custom event-handler vm.on_output("UDP test service", UDP_test) From 0e61a916f6096300a128f4de7baed281866c3351 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Thu, 4 Oct 2018 16:37:13 +0200 Subject: [PATCH 701/723] test: Send mega-sized datagram to IncludeOS and back --- src/net/tcp/listener.cpp | 2 +- test/net/integration/udp/CMakeLists.txt | 3 +-- test/net/integration/udp/service.cpp | 32 ++++++++++++------------- test/net/integration/udp/test.py | 22 +++++++++++++++-- 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/src/net/tcp/listener.cpp b/src/net/tcp/listener.cpp index c2d4940886..8b6d65dfb3 100644 --- a/src/net/tcp/listener.cpp +++ b/src/net/tcp/listener.cpp @@ -22,7 +22,7 @@ using namespace net; using namespace tcp; -#define VERBOSE_TCP_LISTENER +//#define VERBOSE_TCP_LISTENER #ifdef VERBOSE_TCP_LISTENER #define TCPL_PRINT1(x, ...) printf(x, ##__VA_ARGS__) #define TCPL_PRINT2(x, ...) printf(x, ##__VA_ARGS__) diff --git a/test/net/integration/udp/CMakeLists.txt b/test/net/integration/udp/CMakeLists.txt index 3ee122c4b3..677412c6b4 100644 --- a/test/net/integration/udp/CMakeLists.txt +++ b/test/net/integration/udp/CMakeLists.txt @@ -11,9 +11,8 @@ project (test_udp) set(SERVICE_NAME "UDP test service") set(BINARY "test_udp") -set(MAX_MEM 128) set(SOURCES service.cpp) -set(DRIVERS virtionet) +set(DRIVERS virtionet ip4_reassembly) # include service build script include($ENV{INCLUDEOS_PREFIX}/includeos/post.service.cmake) diff --git a/test/net/integration/udp/service.cpp b/test/net/integration/udp/service.cpp index 4ee1ce913c..c4a00df8ff 100644 --- a/test/net/integration/udp/service.cpp +++ b/test/net/integration/udp/service.cpp @@ -17,8 +17,6 @@ #include #include -#include - using namespace net; void Service::start() @@ -27,10 +25,7 @@ void Service::start() inet.network_config({ 10, 0, 0, 55 }, // IP { 255, 255, 0, 0 }, // Netmask { 10, 0, 0, 1 } ); // Gateway - printf("Service IP address is %s\n", inet.ip_addr().to_string().c_str()); - - // UDP - const UDP::port_t port = 4242; + const uint16_t port = 4242; auto& sock = inet.udp().bind(port); sock.on_read( @@ -38,17 +33,20 @@ void Service::start() UDP::addr_t addr, UDP::port_t port, const char* data, size_t len) { - std::string strdata(data, len); - CHECK(1, "Getting UDP data from %s: %d -> %s", - addr.to_string().c_str(), port, strdata.c_str()); - // send the same thing right back! - using namespace std::chrono; - Timers::oneshot(100ms, - Timers::handler_t::make_packed( - [&sock, addr, port, data, len](Timers::id_t) { - sock.sendto(addr, port, data, len); - })); - }); + CHECK(1, "UDP data from %s:%u -> %.*s", + addr.to_string().c_str(), port, std::min((int) len, 16), data); + // send the same thing right back! + printf("Got %lu bytes\n", len); + while (len >= 1472) + { + sock.sendto(addr, port, data, 1472); + data += 1472; + len -= 1472; + } + if (len > 0) { + sock.sendto(addr, port, data, len); + } + }); INFO("UDP test", "Listening on port %d\n", port); } diff --git a/test/net/integration/udp/test.py b/test/net/integration/udp/test.py index d7efddea51..89ae738982 100755 --- a/test/net/integration/udp/test.py +++ b/test/net/integration/udp/test.py @@ -40,8 +40,26 @@ def UDP_test(trigger_line): print " Sent: {}".format(data) print " Received: {}".format(received) - if received == data: - vmrunner.vms[0].exit(0, "Test completed without errors") + if received != data: return False + + data = "x" * 1472 + sock.sendto(data, (HOST, PORT)) + received = sock.recv(1500) + if received != data: + print " Did not receive long string: {}".format(received) + return False + + data = "x" * 32000 + sock.sendto(data, (HOST, PORT)) + received = bytearray() + while (len(received) < len(data)): + received.extend(sock.recv(len(data))) + print "RECEIVED: ", len(received) + if received != data: + print " Did not receive mega string (64k)" + return False + + vmrunner.vms[0].exit(0, "Test completed without errors") # Add custom event-handler vm.on_output("UDP test service", UDP_test) From 9ed7613697e4198720e45097f425a1db299ecb68 Mon Sep 17 00:00:00 2001 From: Martin Nordsletten Date: Fri, 5 Oct 2018 10:46:38 +0200 Subject: [PATCH 702/723] Create_bridge: Remove double parantheseses as they are not sh compatible --- etc/scripts/create_bridge.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/scripts/create_bridge.sh b/etc/scripts/create_bridge.sh index deef29cf20..7b8bf2df05 100755 --- a/etc/scripts/create_bridge.sh +++ b/etc/scripts/create_bridge.sh @@ -33,7 +33,7 @@ if [ -n "$INCLUDEOS_BRIDGE" ]; then fi echo " Creating bridge $BRIDGE, netmask $NETMASK, gateway $GATEWAY " -if [[ -n "$GATEWAY6" ]]; then +if [ -n "$GATEWAY6" ]; then echo " ipv6 netmask $NETMASK6, gateway $GATEWAY6 " fi @@ -89,7 +89,7 @@ else echo " Configuring network bridge (requires sudo):" sudo ifconfig $BRIDGE $GATEWAY netmask $NETMASK up || exit 1 - if [[ -n "$GATEWAY6" ]]; then + if [ -n "$GATEWAY6" ]; then sudo ifconfig $BRIDGE inet6 add $GATEWAY6/$NETMASK6 fi if uname -s | grep Darwin > /dev/null 2>&1; then From 37853013e7d0c69574422aa642dad142f3ace9a4 Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Fri, 5 Oct 2018 10:59:27 +0200 Subject: [PATCH 703/723] Updating NaCl (updating iface and vlan functionality) --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index ab869f7a0f..7dc3578a8e 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit ab869f7a0fd0942fbb808edd1bb376481e1f866f +Subproject commit 7dc3578a8e5160c50ed4c311353bee1e9f62219d From 8304fb8b49f81fa68dfe2f6d5a1280667d9eadcc Mon Sep 17 00:00:00 2001 From: Annika Hammervoll Date: Fri, 5 Oct 2018 11:10:23 +0200 Subject: [PATCH 704/723] Updating NaCl (timers.cpp goldenfile) --- NaCl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NaCl b/NaCl index 7dc3578a8e..a5e919de6c 160000 --- a/NaCl +++ b/NaCl @@ -1 +1 @@ -Subproject commit 7dc3578a8e5160c50ed4c311353bee1e9f62219d +Subproject commit a5e919de6c8afa58ef9098ad65e7437a72a97ad2 From 4593fc9a557667241baf95d87f989bd2a8333cf5 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 5 Oct 2018 11:36:17 +0200 Subject: [PATCH 705/723] test: reduce large datagram size to MacOS default max in UDP test --- test/net/integration/udp/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/net/integration/udp/test.py b/test/net/integration/udp/test.py index 89ae738982..3eac4ab44d 100755 --- a/test/net/integration/udp/test.py +++ b/test/net/integration/udp/test.py @@ -26,7 +26,7 @@ def UDP_test(trigger_line): # been shut down due to a VM timeout sock.settimeout(20) - data = "Douche" + data = "Lucky" sock.sendto(data, (HOST, PORT)) received = sock.recv(1024) @@ -34,7 +34,7 @@ def UDP_test(trigger_line): print " Received: {}".format(received) if received != data: return False - data = "Bag" + data = "Luke" sock.sendto(data, (HOST, PORT)) received = sock.recv(1024) @@ -49,7 +49,7 @@ def UDP_test(trigger_line): print " Did not receive long string: {}".format(received) return False - data = "x" * 32000 + data = "x" * 9216 # 9216 is apparently default max for MacOS sock.sendto(data, (HOST, PORT)) received = bytearray() while (len(received) < len(data)): From 0c6ef50bc2e9ce1e9d12f2e784db0c983d9d2a02 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 5 Oct 2018 11:54:24 +0200 Subject: [PATCH 706/723] kernel: prevent infinite loops of cpu-exceptions + use fprintf always --- src/platform/x86_pc/idt.cpp | 77 +++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 29 deletions(-) diff --git a/src/platform/x86_pc/idt.cpp b/src/platform/x86_pc/idt.cpp index 827efea8cc..4b979eb1c7 100644 --- a/src/platform/x86_pc/idt.cpp +++ b/src/platform/x86_pc/idt.cpp @@ -3,6 +3,8 @@ #include #include #include +#include +#include #define RING0_CODE_SEG 0x8 extern "C" { @@ -229,35 +231,35 @@ void __cpu_dump_regs(uintptr_t* regs) asm ("sidtq %0" : : "m" (* &idt)); fprintf(stderr, "\n"); - printf(" RAX: %016lx R 8: %016lx\n", regs[0], regs[5]); - printf(" RBX: %016lx R 9: %016lx\n", regs[1], regs[6]); - printf(" RCX: %016lx R10: %016lx\n", regs[2], regs[7]); - printf(" RDX: %016lx R11: %016lx\n", regs[3], regs[8]); + fprintf(stderr," RAX: %016lx R 8: %016lx\n", regs[0], regs[5]); + fprintf(stderr," RBX: %016lx R 9: %016lx\n", regs[1], regs[6]); + fprintf(stderr," RCX: %016lx R10: %016lx\n", regs[2], regs[7]); + fprintf(stderr," RDX: %016lx R11: %016lx\n", regs[3], regs[8]); fprintf(stderr, "\n"); - printf(" RBP: %016lx R12: %016lx\n", regs[4], regs[9]); - printf(" RSP: %016lx R13: %016lx\n", regs[13], regs[10]); - printf(" RSI: %016lx R14: %016lx\n", regs[14], regs[11]); - printf(" RDI: %016lx R15: %016lx\n", regs[15], regs[12]); - printf(" RIP: %016lx FLA: %016lx\n", regs[16], regs[17]); + fprintf(stderr," RBP: %016lx R12: %016lx\n", regs[4], regs[9]); + fprintf(stderr," RSP: %016lx R13: %016lx\n", regs[13], regs[10]); + fprintf(stderr," RSI: %016lx R14: %016lx\n", regs[14], regs[11]); + fprintf(stderr," RDI: %016lx R15: %016lx\n", regs[15], regs[12]); + fprintf(stderr," RIP: %016lx FLA: %016lx\n", regs[16], regs[17]); fprintf(stderr, "\n"); - printf(" CR0: %016lx CR4: %016lx\n", regs[18], regs[22]); - printf(" CR1: %016lx CR8: %016lx\n", regs[19], regs[23]); - printf(" CR2: %016lx GDT: %016lx (%u)\n", regs[20], gdt.location, gdt.limit); - printf(" CR3: %016lx IDT: %016lx (%u)\n", regs[21], idt.location, idt.limit); + fprintf(stderr," CR0: %016lx CR4: %016lx\n", regs[18], regs[22]); + fprintf(stderr," CR1: %016lx CR8: %016lx\n", regs[19], regs[23]); + fprintf(stderr," CR2: %016lx GDT: %016lx (%u)\n", regs[20], gdt.location, gdt.limit); + fprintf(stderr," CR3: %016lx IDT: %016lx (%u)\n", regs[21], idt.location, idt.limit); #elif defined(ARCH_i686) # define RIP_REG 8 // i386 CPU registers fprintf(stderr, "\n"); - printf(" EAX: %08x EBP: %08x\n", regs[0], regs[4]); - printf(" EBX: %08x ESP: %08x\n", regs[1], regs[5]); - printf(" ECX: %08x ESI: %08x\n", regs[2], regs[6]); - printf(" EDX: %08x EDI: %08x\n", regs[3], regs[7]); + fprintf(stderr," EAX: %08x EBP: %08x\n", regs[0], regs[4]); + fprintf(stderr," EBX: %08x ESP: %08x\n", regs[1], regs[5]); + fprintf(stderr," ECX: %08x ESI: %08x\n", regs[2], regs[6]); + fprintf(stderr," EDX: %08x EDI: %08x\n", regs[3], regs[7]); fprintf(stderr, "\n"); - printf(" EIP: %08x EFL: %08x\n", regs[8], regs[9]); + fprintf(stderr," EIP: %08x EFL: %08x\n", regs[8], regs[9]); #else #error "Unknown architecture" @@ -269,42 +271,58 @@ extern "C" void double_fault(const char*); void __page_fault(uintptr_t* regs, uint32_t code) { const char* reason = "Protection violation"; + auto addr = regs[20]; if (not(code & 1)) reason = "Page not present"; - kprintf("%s, trying to access 0x%lx\n", reason, regs[20]); + fprintf(stderr,"%s, trying to access 0x%lx\n", reason, addr); if (code & 2) - kprintf("Page write failed.\n"); + fprintf(stderr,"Page write failed.\n"); else - kprintf("Page read failed.\n"); + fprintf(stderr,"Page read failed.\n"); if (code & 4) - kprintf("Privileged page access from user space.\n"); + fprintf(stderr,"Privileged page access from user space.\n"); if (code & 8) - kprintf("Found bit set in reserved field.\n"); + fprintf(stderr,"Found bit set in reserved field.\n"); if (code & 16) - kprintf("Instruction fetch. XD\n"); + fprintf(stderr,"Instruction fetch. XD\n"); if (code & 32) - kprintf("Protection key violation.\n"); + fprintf(stderr,"Protection key violation.\n"); if (code & 0x8000) - kprintf("SGX access violation.\n"); + fprintf(stderr,"SGX access violation.\n"); + + auto key = OS::memory_map().in_range(addr); + if (key) { + auto& range = OS::memory_map().at(key); + printf("Violated address is in mapped range \"%s\" \n", range.name()); + } else { + printf("Violated ddress is outside mapped memory\n"); + } + } +static int exception_counter = 0; + extern "C" __attribute__((noreturn, weak)) void __cpu_exception(uintptr_t* regs, int error, uint32_t code) { - cpu_enable_panicking(); + __sync_fetch_and_add(&exception_counter, 1); + if (exception_counter > 1) { + panic("Double CPU exception"); + } + SMP::global_lock(); - kprintf("\n>>>> !!! CPU %u EXCEPTION !!! <<<<\n", SMP::cpu_id()); - kprintf("%s (%d) EIP %p CODE %#x\n", + fprintf(stderr,"\n>>>> !!! CPU %u EXCEPTION !!! <<<<\n", SMP::cpu_id()); + fprintf(stderr,"%s (%d) EIP %p CODE %#x\n", exception_names[error], error, (void*) regs[RIP_REG], code); if (error == PAGE_FAULT) { @@ -317,6 +335,7 @@ void __cpu_exception(uintptr_t* regs, int error, uint32_t code) // error message: char buffer[64]; snprintf(buffer, sizeof(buffer), "%s (%d)", exception_names[error], error); + // normal CPU exception if (error != 0x8) { // call panic, which will decide what to do next From e50b13b3e95d85ccef320daa8ad657c8067391af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Mon, 8 Oct 2018 12:48:24 +0200 Subject: [PATCH 707/723] http: Changes to Client options, no longer takes bufsize --- api/net/http/basic_client.hpp | 16 +++------------- api/net/http/client_connection.hpp | 4 ++-- lib/uplink/ws_uplink.cpp | 4 +++- src/net/http/basic_client.cpp | 2 +- src/net/http/client_connection.cpp | 12 ++++++------ test/net/integration/http/service.cpp | 4 +++- 6 files changed, 18 insertions(+), 24 deletions(-) diff --git a/api/net/http/basic_client.hpp b/api/net/http/basic_client.hpp index 17fec35dff..7712ee70e1 100644 --- a/api/net/http/basic_client.hpp +++ b/api/net/http/basic_client.hpp @@ -50,24 +50,14 @@ namespace http { using timeout_duration = Client_connection::timeout_duration; const static timeout_duration DEFAULT_TIMEOUT; // client.cpp, 5s - constexpr static size_t DEFAULT_BUFSIZE = 2048; /* Client Options */ - // if someone has a better solution, please fix + // aggregate initialization would make this pretty (c++20): + // https://en.cppreference.com/w/cpp/language/aggregate_initialization struct Options { timeout_duration timeout{DEFAULT_TIMEOUT}; - size_t bufsize{DEFAULT_BUFSIZE}; - Options(timeout_duration dur, size_t bufsz) - : timeout{dur}, - bufsize{bufsz} - {} - - Options() : Options(DEFAULT_TIMEOUT, DEFAULT_BUFSIZE) {} - - Options(timeout_duration dur) : Options(dur, DEFAULT_BUFSIZE) {} - - Options(size_t bufsz) : Options(DEFAULT_TIMEOUT, bufsz) {} + Options() noexcept {} }; diff --git a/api/net/http/client_connection.hpp b/api/net/http/client_connection.hpp index 3345ef052a..7e5e7a13e9 100644 --- a/api/net/http/client_connection.hpp +++ b/api/net/http/client_connection.hpp @@ -44,7 +44,7 @@ namespace http { bool occupied() const { return !available(); } - void send(Request_ptr, Response_handler, const size_t bufsize, timeout_duration = timeout_duration::zero()); + void send(Request_ptr, Response_handler, timeout_duration = timeout_duration::zero()); private: Basic_client& client_; @@ -54,7 +54,7 @@ namespace http { Timer timer_; timeout_duration timeout_dur_; - void send_request(const size_t bufsize); + void send_request(); void recv_response(buffer_t buf); diff --git a/lib/uplink/ws_uplink.cpp b/lib/uplink/ws_uplink.cpp index 47c9a5a019..6ba7a151ce 100644 --- a/lib/uplink/ws_uplink.cpp +++ b/lib/uplink/ws_uplink.cpp @@ -190,12 +190,14 @@ namespace uplink { url << endpoint; MYINFO("[ %s ] Sending auth request to %s", isotime::now().c_str(), url.to_string().c_str()); + http::Basic_client::Options options; + options.timeout = 15s; client_->post(url, { {"Content-Type", "application/json"} }, auth_data(), {this, &WS_uplink::handle_auth_response}, - http::Basic_client::Options{15s}); + options); } void WS_uplink::handle_auth_response(http::Error err, http::Response_ptr res, http::Connection&) diff --git a/src/net/http/basic_client.cpp b/src/net/http/basic_client.cpp index 2b72ed5cae..c241518712 100644 --- a/src/net/http/basic_client.cpp +++ b/src/net/http/basic_client.cpp @@ -68,7 +68,7 @@ namespace http { if(on_send_) on_send_(*req, options, host); - conn.send(move(req), move(cb), options.bufsize, options.timeout); + conn.send(move(req), move(cb), options.timeout); } void Basic_client::request(Method method, URI url, Header_set hfields, diff --git a/src/net/http/client_connection.cpp b/src/net/http/client_connection.cpp index eb2773239f..ddc7998e5e 100644 --- a/src/net/http/client_connection.cpp +++ b/src/net/http/client_connection.cpp @@ -33,7 +33,7 @@ namespace http { stream_->on_close({this, &Client_connection::close}); } - void Client_connection::send(Request_ptr req, Response_handler on_res, const size_t bufsize, timeout_duration timeout) + void Client_connection::send(Request_ptr req, Response_handler on_res, timeout_duration timeout) { Expects(available()); req_ = std::move(req); @@ -47,21 +47,21 @@ namespace http { // if the stream is not established, send the request when connected if(not stream_->is_connected()) { - stream_->on_connect([this, bufsize](auto&) + stream_->on_connect([this](auto&) { - this->send_request(bufsize); + this->send_request(); }); } else { - send_request(bufsize); + send_request(); } } - void Client_connection::send_request(const size_t bufsize) + void Client_connection::send_request() { keep_alive_ = (req_->header().value(header::Connection) != "close"); - stream_->on_read(bufsize, {this, &Client_connection::recv_response}); + stream_->on_read(0, {this, &Client_connection::recv_response}); stream_->write(req_->to_string()); } diff --git a/test/net/integration/http/service.cpp b/test/net/integration/http/service.cpp index 44303b66f2..31c0d9d7f8 100644 --- a/test/net/integration/http/service.cpp +++ b/test/net/integration/http/service.cpp @@ -98,6 +98,8 @@ void Service::ready() }); using namespace std::chrono; + Basic_client::Options options; + options.timeout = 3s; client_->get(acorn_url + "api/dashboard/status", {}, [] (Error err, Response_ptr res, Connection&) { @@ -105,7 +107,7 @@ void Service::ready() CHECK(res != nullptr, "Received response"); if(!err) printf("Response:\n%s\n", res->to_string().c_str()); - }, { 3s }); + }, options); INFO("Basic_client", "HTTPS"); From c3ded62abb64a67acc22f62e9eb7a0af3bf3815c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=85kesson?= Date: Tue, 9 Oct 2018 14:28:17 +0200 Subject: [PATCH 708/723] http: Support follow redirection in Client #1727 --- api/net/http/basic_client.hpp | 13 +++++++- api/net/http/client_connection.hpp | 7 +++- examples/http_client/service.cpp | 27 +++++++++++++++ src/net/http/basic_client.cpp | 49 ++++++++++++++++++++++++++- src/net/http/client_connection.cpp | 53 +++++++++++++++++++++++++++--- 5 files changed, 142 insertions(+), 7 deletions(-) diff --git a/api/net/http/basic_client.hpp b/api/net/http/basic_client.hpp index 7712ee70e1..c088bf5153 100644 --- a/api/net/http/basic_client.hpp +++ b/api/net/http/basic_client.hpp @@ -50,12 +50,14 @@ namespace http { using timeout_duration = Client_connection::timeout_duration; const static timeout_duration DEFAULT_TIMEOUT; // client.cpp, 5s + static int default_follow_redirect; // 0 /* Client Options */ // aggregate initialization would make this pretty (c++20): // https://en.cppreference.com/w/cpp/language/aggregate_initialization struct Options { timeout_duration timeout{DEFAULT_TIMEOUT}; + int follow_redirect{default_follow_redirect}; Options() noexcept {} @@ -86,6 +88,15 @@ namespace http { void send(Request_ptr req, Host host, Response_handler cb, const bool secure = false, Options options = {}); + /** + * @brief Send a request to a specific URL with a response handler + * + * @param[in] req The request + * @param[in] url The URL + * @param[in] cb Callback to be invoked when a response is received (or error) + */ + void send(Request_ptr req, URI url, Response_handler cb, Options options = {}); + /** * @brief Create a request on the given URL * @@ -205,7 +216,7 @@ namespace http { } /** Set uri and Host from URL */ - void populate_from_url(Request& req, const URI& url); + static void populate_from_url(Request& req, const URI& url); /** Add data and content length */ void add_data(Request&, const std::string& data); diff --git a/api/net/http/client_connection.hpp b/api/net/http/client_connection.hpp index 7e5e7a13e9..c71f5499f2 100644 --- a/api/net/http/client_connection.hpp +++ b/api/net/http/client_connection.hpp @@ -44,7 +44,7 @@ namespace http { bool occupied() const { return !available(); } - void send(Request_ptr, Response_handler, timeout_duration = timeout_duration::zero()); + void send(Request_ptr, Response_handler, int redirects, timeout_duration = timeout_duration::zero()); private: Basic_client& client_; @@ -53,6 +53,7 @@ namespace http { Response_handler on_response_; Timer timer_; timeout_duration timeout_dur_; + int redirect_; void send_request(); @@ -63,6 +64,10 @@ namespace http { void timeout_request() { end_response(Error::TIMEOUT); } + bool can_redirect(const Response_ptr&) const; + + void redirect(uri::URI url); + void close() override; }; // < class Client_connection diff --git a/examples/http_client/service.cpp b/examples/http_client/service.cpp index 1edd8717af..f64a0b0e1e 100644 --- a/examples/http_client/service.cpp +++ b/examples/http_client/service.cpp @@ -76,6 +76,33 @@ static void begin_http(net::Inet& inet) printf("Make sure the virtual machine can reach internet.\n"); } }); + + Client::Options options; + options.follow_redirect = 0; + const auto url_mis{"https://www.facebok.com"s}; + client.get(url_mis, {}, [url = url_mis](Error err, Response_ptr res, Connection&) + { + if(not err) { + std::cout << "\n" << url << " - Got response!\n" << res->status_line() << "\n" << res->header() << "\n"; + } + else { + printf("\n%s - No response: %s\n", url.c_str(), err.to_string().c_str()); + printf("Make sure the virtual machine can reach internet.\n"); + } + }, options); + + options.follow_redirect = 1; + client.get(url_mis, {}, [url = url_mis](Error err, Response_ptr res, Connection&) + { + if(not err) { + std::cout << "\n" << url << " - Got response!\n" << res->status_line() << "\n" << res->header() << "\n"; + } + else { + printf("\n%s - No response: %s\n", url.c_str(), err.to_string().c_str()); + printf("Make sure the virtual machine can reach internet.\n"); + } + }, options); + } void Service::start() diff --git a/src/net/http/basic_client.cpp b/src/net/http/basic_client.cpp index c241518712..35cfa526c5 100644 --- a/src/net/http/basic_client.cpp +++ b/src/net/http/basic_client.cpp @@ -20,6 +20,7 @@ namespace http { const Basic_client::timeout_duration Basic_client::DEFAULT_TIMEOUT{std::chrono::seconds(5)}; + int Basic_client::default_follow_redirect{0}; Basic_client::Basic_client(TCP& tcp, Request_handler on_send) : Basic_client(tcp, std::move(on_send), false) @@ -68,7 +69,53 @@ namespace http { if(on_send_) on_send_(*req, options, host); - conn.send(move(req), move(cb), options.timeout); + conn.send(move(req), move(cb), options.follow_redirect, options.timeout); + } + + void Basic_client::send(Request_ptr req, URI url, Response_handler cb, Options options) + { + // find out if this is a secured request or not + const bool secure = url.scheme_is_secure(); + validate_secure(secure); + + using namespace std; + + // Default to port 80 if non given + const uint16_t port = (url.port() != 0xFFFF) ? url.port() : 80; + + if (url.host_is_ip4()) + { + std::string host(url.host()); + auto ip = net::ip4::Addr(host); + + send(move(req), {ip, port}, move(cb), secure, move(options)); + } + else + { + tcp_.stack().resolve(std::string(url.host()), + ResolveCallback::make_packed( + [ + this, + request = move(req), + url{move(url)}, + cb{move(cb)}, + opt{move(options)}, + secure, + port + ] + (net::ip4::Addr ip, const net::Error&) mutable + { + // Host resolved + if (ip != 0) + { + send(move(request), {ip, port}, move(cb), secure, move(opt)); + } + else + { + cb({Error::RESOLVE_HOST}, nullptr, Connection::empty()); + } + })); + } } void Basic_client::request(Method method, URI url, Header_set hfields, diff --git a/src/net/http/client_connection.cpp b/src/net/http/client_connection.cpp index ddc7998e5e..1e1c5a860a 100644 --- a/src/net/http/client_connection.cpp +++ b/src/net/http/client_connection.cpp @@ -27,19 +27,21 @@ namespace http { res_(nullptr), on_response_{nullptr}, timer_({this, &Client_connection::timeout_request}), - timeout_dur_{timeout_duration::zero()} + timeout_dur_{timeout_duration::zero()}, + redirect_{client.default_follow_redirect} { // setup close event stream_->on_close({this, &Client_connection::close}); } - void Client_connection::send(Request_ptr req, Response_handler on_res, timeout_duration timeout) + void Client_connection::send(Request_ptr req, Response_handler on_res, int redirects, timeout_duration timeout) { Expects(available()); req_ = std::move(req); on_response_ = std::move(on_res); Expects(on_response_ != nullptr); timeout_dur_ = timeout; + redirect_ = redirects; if(timeout_dur_ > timeout_duration::zero()) timer_.restart(timeout_dur_); @@ -61,7 +63,7 @@ namespace http { { keep_alive_ = (req_->header().value(header::Connection) != "close"); - stream_->on_read(0, {this, &Client_connection::recv_response}); + stream_->on_read(0 , {this, &Client_connection::recv_response}); stream_->write(req_->to_string()); } @@ -150,7 +152,20 @@ namespace http { // just discard (we can't do anything because we have no callback). // This MAY have side effects if the connection is reused and // a delayed response is received - if (on_response_) { + if (on_response_) + { + if(UNLIKELY(not err and can_redirect(res_))) + { + uri::URI location{res_->header().value("Location")}; + + if(location.is_valid()) + { + redirect(location); + end(); // hope its good here.. + return; + } + + } // move response to a copy in case of callback result in new request auto callback = std::move(on_response_); on_response_.reset(); @@ -175,6 +190,36 @@ namespace http { }*/ } + bool Client_connection::can_redirect(const Response_ptr& res) const + { + if(redirect_ < 1) return false; + if(res == nullptr) return false; + if(not is_redirection(res->status_code())) return false; + if(not res->header().has_field("Location")) return false; + + return true; + } + + void Client_connection::redirect(uri::URI location) + { + Expects(on_response_); + + // modify req + client_.populate_from_url(*req_, location); + + // build option + Basic_client::Options options; + options.timeout = timeout_dur_; + options.follow_redirect = --redirect_; + + // move callback + auto callback = std::move(on_response_); + on_response_.reset(); + + // have client send new request + client_.send(std::move(req_), location, std::move(callback), options); + } + void Client_connection::close() { // if the user havent received a response yet From 728b1e23e9a28809c06883cfac99ef1a6f33c58f Mon Sep 17 00:00:00 2001 From: taiyeba Date: Tue, 9 Oct 2018 15:37:17 +0200 Subject: [PATCH 709/723] test: fs,net,posix,stress test.py has been edited to use subprocess32 with a thread_timeout --- test/fs/integration/fat32/test.py | 7 +++++-- test/fs/integration/ide/test.py | 7 +++++-- test/fs/integration/ide_write/test.py | 7 +++++-- test/fs/integration/vfs/test.py | 9 ++++++--- test/fs/integration/virtio_block/test.py | 7 +++++-- test/net/integration/dhclient/test.py | 8 +++++--- test/net/integration/dhcpd/test.py | 7 +++++-- .../net/integration/dhcpd_dhclient_linux/test.py | 16 ++++++++++------ test/net/integration/dns/test.py | 4 +++- test/net/integration/icmp/test.py | 9 ++++++--- test/net/integration/router/test.py | 12 ++++++++---- test/posix/integration/syslog_plugin/test.py | 4 +++- test/stress/test.py | 13 +++++++------ 13 files changed, 73 insertions(+), 37 deletions(-) diff --git a/test/fs/integration/fat32/test.py b/test/fs/integration/fat32/test.py index 35ddb0133e..da6de50223 100755 --- a/test/fs/integration/fat32/test.py +++ b/test/fs/integration/fat32/test.py @@ -2,6 +2,9 @@ import sys import os import subprocess +import subprocess32 + +thread_timeout = 30 includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) @@ -17,10 +20,10 @@ def cleanup(): print subprocess.check_output(["./fat32_disk.sh", "clean"]) # Setup disk -subprocess.call(["./fat32_disk.sh"], shell=True) +subprocess32.call(["./fat32_disk.sh"], shell=True, timeout=thread_timeout) # Clean up on exit vm.on_exit(cleanup) # Boot the VM -vm.cmake().boot(30).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/fs/integration/ide/test.py b/test/fs/integration/ide/test.py index 35ddb0133e..da6de50223 100755 --- a/test/fs/integration/ide/test.py +++ b/test/fs/integration/ide/test.py @@ -2,6 +2,9 @@ import sys import os import subprocess +import subprocess32 + +thread_timeout = 30 includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) @@ -17,10 +20,10 @@ def cleanup(): print subprocess.check_output(["./fat32_disk.sh", "clean"]) # Setup disk -subprocess.call(["./fat32_disk.sh"], shell=True) +subprocess32.call(["./fat32_disk.sh"], shell=True, timeout=thread_timeout) # Clean up on exit vm.on_exit(cleanup) # Boot the VM -vm.cmake().boot(30).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/fs/integration/ide_write/test.py b/test/fs/integration/ide_write/test.py index 35ddb0133e..fdae204697 100755 --- a/test/fs/integration/ide_write/test.py +++ b/test/fs/integration/ide_write/test.py @@ -2,6 +2,9 @@ import sys import os import subprocess +import subprocess32 + +thread_timeout = 30 includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) @@ -17,10 +20,10 @@ def cleanup(): print subprocess.check_output(["./fat32_disk.sh", "clean"]) # Setup disk -subprocess.call(["./fat32_disk.sh"], shell=True) +subprocess32.call(["./fat32_disk.sh"], shell=True, timeout = thread_timeout) # Clean up on exit vm.on_exit(cleanup) # Boot the VM -vm.cmake().boot(30).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/fs/integration/vfs/test.py b/test/fs/integration/vfs/test.py index 381a399b1a..5e2a6b0bd5 100755 --- a/test/fs/integration/vfs/test.py +++ b/test/fs/integration/vfs/test.py @@ -2,6 +2,9 @@ import sys import subprocess import os +import subprocess32 + +thread_timeout = 20 includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) @@ -16,12 +19,12 @@ def cleanup(): for disk in disks: diskname = disk + ".disk" print "Removing disk file ", diskname - subprocess.check_call(["rm", "-f", diskname]) + subprocess32.check_call(["rm", "-f", diskname], timeout=thread_timeout) # Create all data disk images from folder names for disk in disks: - subprocess.check_call(["./create_disk.sh", disk, disk]) + subprocess32.check_call(["./create_disk.sh", disk, disk]) vmrunner.vms[0].on_exit_success(cleanup) -vmrunner.vms[0].cmake().boot(20).clean() +vmrunner.vms[0].cmake().boot(thread_timeout).clean() diff --git a/test/fs/integration/virtio_block/test.py b/test/fs/integration/virtio_block/test.py index 3a19137a4e..26fd0d0832 100755 --- a/test/fs/integration/virtio_block/test.py +++ b/test/fs/integration/virtio_block/test.py @@ -1,12 +1,15 @@ #! /usr/bin/env python import sys import subprocess +import subprocess32 import os +thread_timeout = 50 + includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) sys.path.insert(0,includeos_src) -subprocess.call(['./image.sh']) +subprocess32.call(['./image.sh'], timeout=thread_timeout) def cleanup(): subprocess.call(['./cleanup.sh']) @@ -15,4 +18,4 @@ def cleanup(): vm = vmrunner.vms[0] vm.on_exit(cleanup) -vm.cmake().boot(50).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/net/integration/dhclient/test.py b/test/net/integration/dhclient/test.py index 0f7865e6c5..444199e4da 100755 --- a/test/net/integration/dhclient/test.py +++ b/test/net/integration/dhclient/test.py @@ -4,6 +4,9 @@ import os import time import subprocess +import subprocess32 + +thread_timeout = 20 includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) @@ -28,16 +31,15 @@ def DHCP_test(trigger_line): try: command = ["ping", ip_string.rstrip(), "-c", str(ping_count), "-i", "0.2"] print color.DATA(" ".join(command)) - print subprocess.check_output(command) + print subprocess32.check_output(command, timeout=thread_timeout) vm.exit(0," Ping test passed. Process returned 0 exit status") except Exception as e: print color.FAIL(" Ping FAILED Process threw exception:") print e return False - # Add custom event-handler vm.on_output("Got IP from DHCP", DHCP_test) # Boot the VM, taking a timeout as parameter -vm.cmake().boot(20).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/net/integration/dhcpd/test.py b/test/net/integration/dhcpd/test.py index 563f2ba2fa..6986374e0d 100755 --- a/test/net/integration/dhcpd/test.py +++ b/test/net/integration/dhcpd/test.py @@ -4,6 +4,9 @@ import os import time import subprocess +import subprocess32 + +thread_timeout = 20 includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) @@ -29,7 +32,7 @@ def DHCP_test(trigger_line): try: command = ["ping", "-c", str(ping_count), "-i", "0.2", ip_string.rstrip()] print color.DATA(" ".join(command)) - print subprocess.check_output(command) + print subprocess32.check_output(command, timeout=thread_timeout) print color.INFO(""), "Number of ping tests passed: ", str(num_assigned_clients) if num_assigned_clients == 3: vm.exit(0," Ping test for all 3 clients passed. Process returned 0 exit status") @@ -44,4 +47,4 @@ def DHCP_test(trigger_line): vm.on_output("Client 3 got IP from IncludeOS DHCP server", DHCP_test) # Boot the VM, taking a timeout as parameter -vm.cmake().boot(20).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/net/integration/dhcpd_dhclient_linux/test.py b/test/net/integration/dhcpd_dhclient_linux/test.py index 1c00d9a18a..ee12af763d 100755 --- a/test/net/integration/dhcpd_dhclient_linux/test.py +++ b/test/net/integration/dhcpd_dhclient_linux/test.py @@ -4,6 +4,9 @@ import os import time import subprocess +import subprocess32 + +thread_timeout = 20 from threading import Timer @@ -69,7 +72,7 @@ def ping_test(): try: command = ["ping", assigned_ip, "-c", str(ping_count), "-i", "0.2"] print color.DATA(" ".join(command)) - print subprocess.check_output(command) + print subprocess32.check_output(command, timeout=thread_timeout) ping_passed = True except Exception as e: print color.FAIL(" Ping FAILED Process threw exception:") @@ -83,20 +86,21 @@ def run_dhclient(trigger_line): route_output = subprocess.check_output(["route"]) if "10.0.0.0" not in route_output: - subprocess.call(["sudo", "ifconfig", "bridge43", "10.0.0.1", "netmask", "255.255.0.0", "up"]) + subprocess32.call(["sudo", "ifconfig", "bridge43", "10.0.0.1", "netmask", "255.255.0.0", "up"], timeout=thread_timeout) time.sleep(1) if "10.200.0.0" not in route_output: - subprocess.call(["sudo", "route", "add", "-net", "10.200.0.0", "netmask", "255.255.0.0", "dev", "bridge43"]) + subprocess32.call(["sudo", "route", "add", "-net", "10.200.0.0", "netmask", "255.255.0.0", "dev", "bridge43"], timeout=thread_timeout) print color.INFO(""), "Route added to bridge43, 10.200.0.0" print color.INFO(""), "Running dhclient" try: - dhclient = subprocess.Popen( + dhclient = subprocess32.Popen( ["sudo", "dhclient", "bridge43", "-4", "-n", "-v"], stdout=subprocess.PIPE, - stderr=subprocess.STDOUT + stderr=subprocess.STDOUT, + timeout=thread_timeout ) # timeout on dhclient process kill_proc = lambda p: p.kill() @@ -128,4 +132,4 @@ def run_dhclient(trigger_line): vm.on_output("Service started", run_dhclient) # Boot the VM, taking a timeout as parameter -vm.cmake().boot(20).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/net/integration/dns/test.py b/test/net/integration/dns/test.py index c584afdb0c..45ead9d123 100755 --- a/test/net/integration/dns/test.py +++ b/test/net/integration/dns/test.py @@ -3,7 +3,9 @@ import sys import os import subprocess +import subprocess32 +thread_timeout = 20 includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) sys.path.insert(0,includeos_src) @@ -36,4 +38,4 @@ def finish(): vm.on_output("some_address_that_doesnt_exist.com couldn't be resolved", count) # Boot the VM, taking a timeout as parameter -vm.cmake().boot(20).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/net/integration/icmp/test.py b/test/net/integration/icmp/test.py index b705386809..e3745bdf88 100755 --- a/test/net/integration/icmp/test.py +++ b/test/net/integration/icmp/test.py @@ -3,6 +3,9 @@ import sys import os import subprocess +import subprocess32 + +thread_timeout = 50 includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) @@ -65,7 +68,7 @@ def start_icmp_test(trigger_line): # 2 Port unreachable print color.INFO(""), "Performing Destination Unreachable (port) test" # Sending 1 udp packet to 10.0.0.45 to port 8080 - udp_port_output = subprocess.check_output(["sudo", "hping3", "10.0.0.45", "--udp", "-p", "8080", "-c", "1"]) + udp_port_output = subprocess32.check_output(["sudo", "hping3", "10.0.0.45", "--udp", "-p", "8080", "-c", "1"], timeout=thread_timeout) print udp_port_output # Validate content in udp_port_output: @@ -78,7 +81,7 @@ def start_icmp_test(trigger_line): # 3 Protocol unreachable print color.INFO(""), "Performing Destination Unreachable (protocol) test" # Sending 1 raw ip packet to 10.0.0.45 with protocol 16 - rawip_protocol_output = subprocess.check_output(["sudo", "hping3", "10.0.0.45", "-d", "20", "-0", "--ipproto", "16", "-c", "1"]) + rawip_protocol_output = subprocess32.check_output(["sudo", "hping3", "10.0.0.45", "-d", "20", "-0", "--ipproto", "16", "-c", "1"], timeout=thread_timeout) print rawip_protocol_output # Validate content in rawip_protocol_output: @@ -99,4 +102,4 @@ def start_icmp_test(trigger_line): vm.on_output("Service IP address is 10.0.0.45", start_icmp_test); # Boot the VM, taking a timeout as parameter -vm.cmake().boot(50).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/net/integration/router/test.py b/test/net/integration/router/test.py index 93ead5303d..7da9c1e1ee 100755 --- a/test/net/integration/router/test.py +++ b/test/net/integration/router/test.py @@ -3,8 +3,11 @@ import sys import os import subprocess +import subprocess32 import thread +thread_timeout = 30 + includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) print 'includeos_src: {0}'.format(includeos_src) @@ -37,14 +40,15 @@ def clean(): def iperf_server(): global iperf_server_proc, iperf_srv_log - iperf_server_proc = subprocess.Popen(["sudo","ip","netns","exec", nsname, iperf_cmd, "-s"], + iperf_server_proc = subprocess32.Popen(["sudo","ip","netns","exec", nsname, iperf_cmd, "-s"], stdout = subprocess.PIPE, stdin = subprocess.PIPE, - stderr = subprocess.PIPE) + stderr = subprocess.PIPE, + timeout=thread_timeout) def iperf_client(o): print "Starting iperf client. Iperf output: " - print subprocess.check_output([iperf_cmd,"-c","10.42.42.2","-n", transmit_size]) + print subprocess32.check_output([iperf_cmd,"-c","10.42.42.2","-n", transmit_size], timeout=thread_timeout) vmrunner.vms[0].exit(0, "Test completed without errors") return True @@ -67,4 +71,4 @@ def iperf_client(o): vm.on_exit(clean) # Boot the VM, taking a timeout as parameter -vm.cmake().boot(30).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/posix/integration/syslog_plugin/test.py b/test/posix/integration/syslog_plugin/test.py index 394d95ae09..4ddb13c112 100755 --- a/test/posix/integration/syslog_plugin/test.py +++ b/test/posix/integration/syslog_plugin/test.py @@ -4,6 +4,8 @@ import subprocess import atexit +thread_timeout = 60 + includeos_src = os.environ.get('INCLUDEOS_SRC', os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))).split('/test')[0]) print 'includeos_src: {0}'.format(includeos_src) @@ -122,4 +124,4 @@ def end(): vm.on_output("Service IP address is 10.0.0.47", start) # Boot the VM, taking a timeout as parameter -vm.cmake().boot(60).clean() +vm.cmake().boot(thread_timeout).clean() diff --git a/test/stress/test.py b/test/stress/test.py index fa58f1889a..8e512a2428 100755 --- a/test/stress/test.py +++ b/test/stress/test.py @@ -3,6 +3,7 @@ import socket import time import subprocess +import subprocess32 import os includeos_src = os.environ.get('INCLUDEOS_SRC', @@ -28,6 +29,9 @@ memuse_at_start = 0 sock_timeout = 20 +# Boot the VM, taking a timeout as parameter +timeout = BURST_COUNT * 30 + # It's to be expected that the VM allocates more room during the running of tests # e.g. for containers, packets etc. These should all be freed after a run. acceptable_increase = 12 * PAGE_SIZE @@ -107,13 +111,13 @@ def UDP_burst(burst_size = BURST_SIZE, burst_interval = BURST_INTERVAL): # Fire a single burst of ICMP packets def ICMP_flood(burst_size = BURST_SIZE, burst_interval = BURST_INTERVAL): # Note: Ping-flooding requires sudo for optimal speed - res = subprocess.check_call(["sudo","ping","-f", HOST, "-c", str(burst_size)]); + res = subprocess32.check_call(["sudo","ping","-f", HOST, "-c", str(burst_size)], timeout=thread_timeout); time.sleep(burst_interval) return get_mem() # Fire a single burst of HTTP requests def httperf(burst_size = BURST_SIZE, burst_interval = BURST_INTERVAL): - res = subprocess.check_call(["httperf","--hog", "--server", HOST, "--num-conn", str(burst_size)]); + res = subprocess32.check_call(["httperf","--hog", "--server", HOST, "--num-conn", str(burst_size)], timeout=thread_timeout); time.sleep(burst_interval) return get_mem() @@ -123,7 +127,7 @@ def ARP_burst(burst_size = BURST_SIZE, burst_interval = BURST_INTERVAL): command = ["sudo", "arping", "-q","-w", str(100), "-I", "bridge43", "-c", str(burst_size * 10), HOST] print color.DATA(" ".join(command)) time.sleep(0.5) - res = subprocess.check_call(command); + res = subprocess32.check_call(command, timeout=thread_timeout); time.sleep(burst_interval) return get_mem() @@ -261,9 +265,6 @@ def wait_for_tw(): vm.on_output("Ready for TCP", TCP) vm.on_output("Ready to end", check_vitals) -# Boot the VM, taking a timeout as parameter -timeout = BURST_COUNT * 30 - if len(sys.argv) > 1: timeout = int(sys.argv[1]) From e72b83b2ae93446b986ff6ba47d93b862f7180a0 Mon Sep 17 00:00:00 2001 From: taiyeba Date: Tue, 9 Oct 2018 15:58:47 +0200 Subject: [PATCH 710/723] =?UTF-8?q?test/stress:=20Changed=20the=20timeout?= =?UTF-8?q?=20parameter=20to=20thread=5Ftimeout=20to=20match=20all=20the?= =?UTF-8?q?=20other=20tests=20and=20also=20to=20avoid=20confusion=20with?= =?UTF-8?q?=20subprocess32=E2=80=99s=20own=20timeout=20parameter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- test/stress/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/stress/test.py b/test/stress/test.py index 8e512a2428..4b35dc3144 100755 --- a/test/stress/test.py +++ b/test/stress/test.py @@ -30,7 +30,7 @@ sock_timeout = 20 # Boot the VM, taking a timeout as parameter -timeout = BURST_COUNT * 30 +thread_timeout = BURST_COUNT * 30 # It's to be expected that the VM allocates more room during the running of tests # e.g. for containers, packets etc. These should all be freed after a run. @@ -266,7 +266,7 @@ def wait_for_tw(): vm.on_output("Ready to end", check_vitals) if len(sys.argv) > 1: - timeout = int(sys.argv[1]) + thread_timeout = int(sys.argv[1]) if len(sys.argv) > 3: BURST_COUNT = int(sys.argv[2]) @@ -275,4 +275,4 @@ def wait_for_tw(): print color.HEADER(test_name + " initializing") print color.INFO(name_tag),"configured for ", BURST_COUNT,"bursts of", BURST_SIZE, "packets each" -vm.cmake().boot(timeout).clean() +vm.cmake().boot(thread_timeout).clean() From 731685f9cd36c2e7785099dbf186fbb9927a0b26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Jerpetj=C3=B8n?= Date: Tue, 9 Oct 2018 23:21:39 +0200 Subject: [PATCH 711/723] run: removed python-magic dependency as this is trublesome and removed --preserve-env booting qemu as it works without ? and requires special sudo requirements on the guest --- vmrunner/vmrunner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vmrunner/vmrunner.py b/vmrunner/vmrunner.py index ed3372e9b0..d79f8e4ff7 100644 --- a/vmrunner/vmrunner.py +++ b/vmrunner/vmrunner.py @@ -10,7 +10,6 @@ import validate_vm import signal import psutil -import magic from shutil import copyfile from prettify import color @@ -83,8 +82,9 @@ def info(*args): def file_type(filename): - with magic.Magic() as m: - return m.id_filename(filename) + p = subprocess.Popen(['file',filename],stdout=subprocess.PIPE,stderr=subprocess.PIPE) + output, errors = p.communicate() + return output def is_Elf64(filename): magic = file_type(filename) @@ -553,7 +553,7 @@ def boot(self, multiboot, debug = False, kernel_args = "", image_name = None): qemu_binary = self._config["qemu"] # TODO: sudo is only required for tap networking and kvm. Check for those. - command = ["sudo", "--preserve-env", qemu_binary] + command = ["sudo", qemu_binary] if self._kvm_present: command.extend(["--enable-kvm"]) command += kernel_args From 63d5bcb1191a2b0ae7a0d046d895327b8550421a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Jerpetj=C3=B8n?= Date: Wed, 10 Oct 2018 09:48:57 +0200 Subject: [PATCH 712/723] Drivers: Added handling of zero packet length in driver --- src/drivers/vmxnet3.cpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index d7e73f4c7c..691b6f36fd 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -514,6 +514,19 @@ bool vmxnet3::receive_handler(const int Q) rx[Q].prod_count--; int desc = comp.index % vmxnet3::NUM_RX_DESC; + + // Handle case of empty packet + if (UNLIKELY((comp.len & (VMXNET3_MAX_BUFFER_LEN-1)) == 0)) { + //TODO assert / log if eop and sop are not set in empty packet. + + //release unused buffer + auto* packet = (net::Packet*) (tx.buffers[desc] - DRIVER_OFFSET - sizeof(net::Packet)); + delete packet; // call deleter on Packet to release it + rx[Q].buffers[desc] = nullptr; + + break; + } + // mask out length int len = comp.len & (VMXNET3_MAX_BUFFER_LEN-1); From b9f2921965f3b20f2b0bbe3c30f6f7a2576cdebd Mon Sep 17 00:00:00 2001 From: taiyeba Date: Wed, 10 Oct 2018 11:03:27 +0200 Subject: [PATCH 713/723] test: removing thread_timeout from router test --- test/net/integration/router/test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/net/integration/router/test.py b/test/net/integration/router/test.py index 7da9c1e1ee..5ec02a4971 100755 --- a/test/net/integration/router/test.py +++ b/test/net/integration/router/test.py @@ -40,11 +40,10 @@ def clean(): def iperf_server(): global iperf_server_proc, iperf_srv_log - iperf_server_proc = subprocess32.Popen(["sudo","ip","netns","exec", nsname, iperf_cmd, "-s"], + iperf_server_proc = subprocess.Popen(["sudo","ip","netns","exec", nsname, iperf_cmd, "-s"], stdout = subprocess.PIPE, stdin = subprocess.PIPE, - stderr = subprocess.PIPE, - timeout=thread_timeout) + stderr = subprocess.PIPE) def iperf_client(o): print "Starting iperf client. Iperf output: " From 92163b3a71781710c063de72996caa84e91612e8 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 10 Oct 2018 11:54:29 +0200 Subject: [PATCH 714/723] statman: Add persistent, gauge/counter flags --- api/util/statman.hpp | 33 ++++++++++++++++++++------------- src/util/statman.cpp | 14 +++++++------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/api/util/statman.hpp b/api/util/statman.hpp index c76ea4d60e..468a1fb4d3 100644 --- a/api/util/statman.hpp +++ b/api/util/statman.hpp @@ -42,6 +42,8 @@ namespace liu { class Stat { public: static const int MAX_NAME_LEN = 46; + static const int GAUGE_BIT = 0x40; + static const int PERSIST_BIT = 0x80; enum Stat_type: uint8_t { @@ -59,14 +61,19 @@ class Stat { void operator++(); Stat_type type() const noexcept - { return type_; } + { return (Stat_type)(m_bits & 0xF); } - const char* name() const noexcept - { return name_; } + bool is_persistent() const noexcept { return m_bits & PERSIST_BIT; } + void make_persistent() noexcept { m_bits |= PERSIST_BIT; } - bool unused() const noexcept { - return name_[0] == 0; - } + bool is_counter() const noexcept { return (m_bits & GAUGE_BIT) == 0; } + bool is_gauge() const noexcept { return (m_bits & GAUGE_BIT) == GAUGE_BIT; } + + void make_counter() noexcept { m_bits &= ~GAUGE_BIT; } + void make_gauge() noexcept { m_bits |= GAUGE_BIT; } + + const char* name() const noexcept { return name_; } + bool unused() const noexcept { return name_[0] == 0; } const float& get_float() const; float& get_float(); @@ -83,7 +90,7 @@ class Stat { uint32_t ui32; uint64_t ui64; }; - Stat_type type_; + uint8_t m_bits; char name_[MAX_NAME_LEN+1]; }; //< class Stat @@ -154,28 +161,28 @@ class Statman { }; //< class Statman inline float& Stat::get_float() { - if (UNLIKELY(type_ != FLOAT)) throw Stats_exception{"Stat type is not a float"}; + if (UNLIKELY(type() != FLOAT)) throw Stats_exception{"Stat type is not a float"}; return f; } inline uint32_t& Stat::get_uint32() { - if (UNLIKELY(type_ != UINT32)) throw Stats_exception{"Stat type is not an uint32"}; + if (UNLIKELY(type() != UINT32)) throw Stats_exception{"Stat type is not an uint32"}; return ui32; } inline uint64_t& Stat::get_uint64() { - if (UNLIKELY(type_ != UINT64)) throw Stats_exception{"Stat type is not an uint64"}; + if (UNLIKELY(type() != UINT64)) throw Stats_exception{"Stat type is not an uint64"}; return ui64; } inline const float& Stat::get_float() const { - if (UNLIKELY(type_ != FLOAT)) throw Stats_exception{"Stat type is not a float"}; + if (UNLIKELY(type() != FLOAT)) throw Stats_exception{"Stat type is not a float"}; return f; } inline const uint32_t& Stat::get_uint32() const { - if (UNLIKELY(type_ != UINT32)) throw Stats_exception{"Stat type is not an uint32"}; + if (UNLIKELY(type() != UINT32)) throw Stats_exception{"Stat type is not an uint32"}; return ui32; } inline const uint64_t& Stat::get_uint64() const { - if (UNLIKELY(type_ != UINT64)) throw Stats_exception{"Stat type is not an uint64"}; + if (UNLIKELY(type() != UINT64)) throw Stats_exception{"Stat type is not an uint64"}; return ui64; } diff --git a/src/util/statman.cpp b/src/util/statman.cpp index 809ee24a7b..ccca714f59 100644 --- a/src/util/statman.cpp +++ b/src/util/statman.cpp @@ -26,7 +26,7 @@ Statman& Statman::get() { } Stat::Stat(const Stat_type type, const std::string& name) - : ui64(0), type_{type} + : ui64(0), m_bits{type} { if (name.size() > MAX_NAME_LEN) throw Stats_exception("Stat name cannot be longer than " + std::to_string(MAX_NAME_LEN) + " characters"); @@ -34,19 +34,19 @@ Stat::Stat(const Stat_type type, const std::string& name) snprintf(name_, sizeof(name_), "%s", name.c_str()); } Stat::Stat(const Stat& other) { - this->ui64 = other.ui64; - this->type_ = other.type_; + this->ui64 = other.ui64; + this->m_bits = other.m_bits; strncpy(this->name_, other.name_, MAX_NAME_LEN); } Stat& Stat::operator=(const Stat& other) { - this->ui64 = other.ui64; - this->type_ = other.type_; + this->ui64 = other.ui64; + this->m_bits = other.m_bits; strncpy(this->name_, other.name_, MAX_NAME_LEN); return *this; } void Stat::operator++() { - switch (type_) { + switch (this->type()) { case UINT32: ui32++; break; case UINT64: ui64++; break; case FLOAT: f += 1.0f; break; @@ -55,7 +55,7 @@ void Stat::operator++() { } std::string Stat::to_string() const { - switch (this->type_) { + switch (this->type()) { case UINT32: return std::to_string(ui32); case UINT64: return std::to_string(ui64); case FLOAT: return std::to_string(f); From 391875092951d8a3339502e47eae77a9692c8815 Mon Sep 17 00:00:00 2001 From: fwsGonzo Date: Wed, 10 Oct 2018 11:54:58 +0200 Subject: [PATCH 715/723] liveupdate: Make it easier to toggle memzero --- lib/LiveUpdate/update.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index 9914ee2948..a8e351890f 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -33,7 +33,6 @@ //#define LPRINT(x, ...) printf(x, ##__VA_ARGS__); #define LPRINT(x, ...) /** x **/ -#define LIU_ZERO_OLD_MEMORY static const int SECT_SIZE = 512; static const int ELF_MINIMUM = 164; @@ -51,6 +50,8 @@ extern char _ELF_START_; extern char _end; // turn this off to reduce liveupdate times at the cost of extra checks bool LIVEUPDATE_PERFORM_SANITY_CHECKS = true; +// turn this om to zero-initialize all memory between new kernel and heap end +bool LIVEUPDATE_ZERO_OLD_MEMORY = false; using namespace liu; @@ -251,14 +252,14 @@ void LiveUpdate::exec(const buffer_t& blob, void* location) // copy hotswapping function to sweet spot memcpy(HOTSWAP_AREA, (void*) &hotswap64, hotswap64_len); /// the end -#ifdef LIU_ZERO_OLD_MEMORY + if (LIVEUPDATE_ZERO_OLD_MEMORY) { ((decltype(&hotswap64)) HOTSWAP_AREA)(phys_base, bin_data, bin_len, start_offset, /* binary entry point */ sr_data, /* softreset location */ (void*) OS::heap_end() /* zero memory until this location */); -#else + } else { ((decltype(&hotswap64)) HOTSWAP_AREA)(phys_base, bin_data, bin_len, start_offset, sr_data, nullptr); -#endif + } # else # error "Unimplemented architecture" # endif From 78b654ed84759dcaaeb42032c34dbfc8a487fea4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Jerpetj=C3=B8n?= Date: Thu, 11 Oct 2018 16:12:25 +0200 Subject: [PATCH 716/723] vmxnet3: added support for adding vlan to nic and apply appropriate filter --- api/hw/nic.hpp | 2 ++ api/net/vlan_manager.hpp | 6 ++++-- src/drivers/vmxnet3.cpp | 7 +++++++ src/drivers/vmxnet3.hpp | 2 ++ src/net/vlan_manager.cpp | 2 +- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/api/hw/nic.hpp b/api/hw/nic.hpp index 23f60f8ba4..1117f5cb97 100644 --- a/api/hw/nic.hpp +++ b/api/hw/nic.hpp @@ -113,6 +113,8 @@ namespace hw { } uint32_t sendq_limit() const noexcept { return m_sendq_limit; } + virtual void add_vlan([[maybe_unused]] const int id){} + protected: /** * Constructor diff --git a/api/net/vlan_manager.hpp b/api/net/vlan_manager.hpp index 55bd8d63b4..32c593608f 100644 --- a/api/net/vlan_manager.hpp +++ b/api/net/vlan_manager.hpp @@ -72,8 +72,10 @@ class VLAN_manager { * * @param link The link */ - void setup(hw::Nic& link) - { link.set_vlan_upstream({this, &VLAN_manager::receive}); } + void setup(hw::Nic& link,const int id) + { link.set_vlan_upstream({this, &VLAN_manager::receive}); + link.add_vlan(id); + } }; diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index d7e73f4c7c..e3d1bfaf42 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -642,6 +642,13 @@ void vmxnet3::poll() this->already_polling = false; } +void vmxnet3::add_vlan(const int id) +{ + this->dma->shared.rx_filter.vlan_filter[((id>>3)&0x1FF)] |= (1<<(id&7)); + // TODO in smp/mt protect this with IRQ disable / enable for the entire driver + mmio_write32(this->iobase + VMXNET3_VD_CMD, VMXNET3_CMD_UPDATE_VLAN_FILTERS); +} + void vmxnet3::deactivate() { // disable all queues diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index 09ec737fee..eb8984dc3e 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -81,6 +81,8 @@ class vmxnet3 : public net::Link_layer void poll() override; + void add_vlan(const int id) override; + private: void msix_evt_handler(); void msix_xmit_handler(); diff --git a/src/net/vlan_manager.cpp b/src/net/vlan_manager.cpp index 9a85f9e639..f9a3102a69 100644 --- a/src/net/vlan_manager.cpp +++ b/src/net/vlan_manager.cpp @@ -51,7 +51,7 @@ VLAN_manager::VLAN_interface& VLAN_manager::add(hw::Nic& link, const int id) // this is very redudant if it's already been set once, // but i'll keep it for now since it's not expensive. - this->setup(link); + this->setup(link,id); auto vif = std::make_unique(link, id); auto* raw = vif.get(); From 9cbf8cdabe65fbcbca510ee2c6092e18dd9b7d23 Mon Sep 17 00:00:00 2001 From: Kristian Jerpetjon Date: Thu, 11 Oct 2018 16:12:25 +0200 Subject: [PATCH 717/723] vmxnet3: added support for adding vlan to nic and apply appropriate filter --- api/hw/nic.hpp | 2 ++ api/net/vlan_manager.hpp | 6 ++++-- src/drivers/vmxnet3.cpp | 7 +++++++ src/drivers/vmxnet3.hpp | 2 ++ src/net/vlan_manager.cpp | 2 +- 5 files changed, 16 insertions(+), 3 deletions(-) diff --git a/api/hw/nic.hpp b/api/hw/nic.hpp index 23f60f8ba4..1117f5cb97 100644 --- a/api/hw/nic.hpp +++ b/api/hw/nic.hpp @@ -113,6 +113,8 @@ namespace hw { } uint32_t sendq_limit() const noexcept { return m_sendq_limit; } + virtual void add_vlan([[maybe_unused]] const int id){} + protected: /** * Constructor diff --git a/api/net/vlan_manager.hpp b/api/net/vlan_manager.hpp index 55bd8d63b4..32c593608f 100644 --- a/api/net/vlan_manager.hpp +++ b/api/net/vlan_manager.hpp @@ -72,8 +72,10 @@ class VLAN_manager { * * @param link The link */ - void setup(hw::Nic& link) - { link.set_vlan_upstream({this, &VLAN_manager::receive}); } + void setup(hw::Nic& link,const int id) + { link.set_vlan_upstream({this, &VLAN_manager::receive}); + link.add_vlan(id); + } }; diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index d7e73f4c7c..e3d1bfaf42 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -642,6 +642,13 @@ void vmxnet3::poll() this->already_polling = false; } +void vmxnet3::add_vlan(const int id) +{ + this->dma->shared.rx_filter.vlan_filter[((id>>3)&0x1FF)] |= (1<<(id&7)); + // TODO in smp/mt protect this with IRQ disable / enable for the entire driver + mmio_write32(this->iobase + VMXNET3_VD_CMD, VMXNET3_CMD_UPDATE_VLAN_FILTERS); +} + void vmxnet3::deactivate() { // disable all queues diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index 09ec737fee..eb8984dc3e 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -81,6 +81,8 @@ class vmxnet3 : public net::Link_layer void poll() override; + void add_vlan(const int id) override; + private: void msix_evt_handler(); void msix_xmit_handler(); diff --git a/src/net/vlan_manager.cpp b/src/net/vlan_manager.cpp index 9a85f9e639..f9a3102a69 100644 --- a/src/net/vlan_manager.cpp +++ b/src/net/vlan_manager.cpp @@ -51,7 +51,7 @@ VLAN_manager::VLAN_interface& VLAN_manager::add(hw::Nic& link, const int id) // this is very redudant if it's already been set once, // but i'll keep it for now since it's not expensive. - this->setup(link); + this->setup(link,id); auto vif = std::make_unique(link, id); auto* raw = vif.get(); From dce50393ef905cfdc8fa74eda01491528bf79f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristian=20Jerpetj=C3=B8n?= Date: Fri, 12 Oct 2018 14:55:59 +0200 Subject: [PATCH 718/723] vmxnet3: Added statistics for zero packet dropped and packets/bytes recieved and sendt --- src/drivers/vmxnet3.cpp | 17 ++++++++++++++++- src/drivers/vmxnet3.hpp | 5 +++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/drivers/vmxnet3.cpp b/src/drivers/vmxnet3.cpp index 691b6f36fd..fc1975dbe4 100644 --- a/src/drivers/vmxnet3.cpp +++ b/src/drivers/vmxnet3.cpp @@ -125,6 +125,14 @@ vmxnet3::vmxnet3(hw::PCI_Device& d, const uint16_t mtu) : m_pcidev(d), m_mtu(mtu), stat_sendq_cur{Statman::get().create(Stat::UINT32, device_name() + ".sendq_now").get_uint32()}, stat_sendq_max{Statman::get().create(Stat::UINT32, device_name() + ".sendq_max").get_uint32()}, + + //TODO make some of these stats generic and put them into LINK object so that they can be seen on all networking drivers + stat_tx_total_packets{Statman::get().create(Stat::UINT64, device_name() + ".stat_tx_total_packets").get_uint64()}, + stat_tx_total_bytes{Statman::get().create(Stat::UINT64, device_name() + ".stat_tx_total_bytes").get_uint64()}, + stat_rx_total_packets{Statman::get().create(Stat::UINT64, device_name() + ".stat_rx_total_packets").get_uint64()}, + stat_rx_total_bytes{Statman::get().create(Stat::UINT64, device_name() + ".stat_rx_total_bytes").get_uint64()}, + stat_rx_zero_dropped{Statman::get().create(Stat::UINT64, device_name() + ".stat_rx_zero_dropped").get_uint64()}, + stat_rx_refill_dropped{Statman::get().create(Stat::UINT64, device_name() + ".rx_refill_dropped").get_uint64()}, stat_sendq_dropped{Statman::get().create(Stat::UINT64, device_name() + ".sendq_dropped").get_uint64()}, bufstore_{1024, buffer_size_for_mtu(mtu)} @@ -523,7 +531,7 @@ bool vmxnet3::receive_handler(const int Q) auto* packet = (net::Packet*) (tx.buffers[desc] - DRIVER_OFFSET - sizeof(net::Packet)); delete packet; // call deleter on Packet to release it rx[Q].buffers[desc] = nullptr; - + stat_rx_zero_dropped++; break; } @@ -533,6 +541,10 @@ bool vmxnet3::receive_handler(const int Q) // get buffer and construct packet assert(rx[Q].buffers[desc] != nullptr); recvq.push_back(recv_packet(rx[Q].buffers[desc], len)); + + stat_rx_total_packets++; + stat_rx_total_bytes+=len; + rx[Q].buffers[desc] = nullptr; } this->enable_intr(2 + Q); @@ -609,6 +621,9 @@ void vmxnet3::transmit_data(uint8_t* data, uint16_t data_length) desc.address = (uintptr_t) tx.buffers[idx]; desc.flags[0] = gen | data_length; desc.flags[1] = VMXNET3_TXF_CQ | VMXNET3_TXF_EOP; + + stat_tx_total_packets++; + stat_tx_total_bytes+=data_length; } void vmxnet3::flush() diff --git a/src/drivers/vmxnet3.hpp b/src/drivers/vmxnet3.hpp index 09ec737fee..368b469de4 100644 --- a/src/drivers/vmxnet3.hpp +++ b/src/drivers/vmxnet3.hpp @@ -143,6 +143,11 @@ class vmxnet3 : public net::Link_layer // sendq as double-ended q uint32_t& stat_sendq_cur; uint32_t& stat_sendq_max; + uint64_t& stat_tx_total_packets; + uint64_t& stat_tx_total_bytes; + uint64_t& stat_rx_total_packets; + uint64_t& stat_rx_total_bytes; + uint64_t& stat_rx_zero_dropped; uint64_t& stat_rx_refill_dropped; uint64_t& stat_sendq_dropped; std::deque sendq; From abbbdabded21b285fd65c284958f5e2d088fd759 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Sat, 13 Oct 2018 23:46:10 +0200 Subject: [PATCH 719/723] x86: Add device deactivation, used by LiveUpdate --- api/arch.hpp | 1 + lib/LiveUpdate/update.cpp | 2 ++ src/platform/kvm/kvmclock.cpp | 9 +++++++-- src/platform/kvm/kvmclock.hpp | 1 + src/platform/x86_pc/clocks.cpp | 3 ++- src/platform/x86_pc/platform.cpp | 10 ++++++++++ src/platform/x86_pc/platform.hpp | 25 +++++++++++++++++++++++++ test/lest_util/os_mock.cpp | 1 + 8 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 src/platform/x86_pc/platform.hpp diff --git a/api/arch.hpp b/api/arch.hpp index 3fb0c0c83c..2e0a5626c4 100644 --- a/api/arch.hpp +++ b/api/arch.hpp @@ -31,6 +31,7 @@ extern void __arch_poweroff(); extern void __arch_reboot(); extern void __arch_enable_legacy_irq(uint8_t); extern void __arch_disable_legacy_irq(uint8_t); +extern void __arch_system_deactivate(); extern void __arch_install_irq(uint8_t, void(*)()); extern void __arch_subscribe_irq(uint8_t); diff --git a/lib/LiveUpdate/update.cpp b/lib/LiveUpdate/update.cpp index a8e351890f..4029faf411 100644 --- a/lib/LiveUpdate/update.cpp +++ b/lib/LiveUpdate/update.cpp @@ -212,6 +212,8 @@ void LiveUpdate::exec(const buffer_t& blob, void* location) // 3. deactivate all devices (eg. mask all MSI-X vectors) // NOTE: there are some nasty side effects from calling this hw::Devices::deactivate_all(); + // turn off devices that affect memory + __arch_system_deactivate(); // store soft-resetting stuff #if defined(PLATFORM_x86_solo5) || defined(PLATFORM_UNITTEST) diff --git a/src/platform/kvm/kvmclock.cpp b/src/platform/kvm/kvmclock.cpp index a91fe395a1..86fc3c7e61 100644 --- a/src/platform/kvm/kvmclock.cpp +++ b/src/platform/kvm/kvmclock.cpp @@ -28,13 +28,13 @@ struct alignas(4096) pvclock_wall_clock { uint32_t nsec; }__attribute__((packed)); static pvclock_wall_clock kvm_wall_clock; +static uint32_t msr_kvm_wall_clock = 0; +static uint32_t msr_kvm_system_time = 0; void KVM_clock::init() { #define MSR_KVM_WALL_CLOCK_NEW 0x4b564d00 #define MSR_KVM_SYSTEM_TIME_NEW 0x4b564d01 - uint32_t msr_kvm_wall_clock; - uint32_t msr_kvm_system_time; if (CPUID::kvm_feature(KVM_FEATURE_CLOCKSOURCE2)) { @@ -55,6 +55,11 @@ void KVM_clock::init() auto vcpu_addr = (uintptr_t) &PER_CPU(vcpu_time); CPU::write_msr(msr_kvm_system_time, vcpu_addr | 1); } +void KVM_clock::deactivate() +{ + CPU::write_msr(msr_kvm_wall_clock, 0); + CPU::write_msr(msr_kvm_system_time, 0); +} KHz KVM_clock::get_tsc_khz() { diff --git a/src/platform/kvm/kvmclock.hpp b/src/platform/kvm/kvmclock.hpp index 1683684be6..e4b87826a2 100644 --- a/src/platform/kvm/kvmclock.hpp +++ b/src/platform/kvm/kvmclock.hpp @@ -9,4 +9,5 @@ struct KVM_clock static uint64_t system_time(); static timespec wall_clock(); static util::KHz get_tsc_khz(); + static void deactivate(); }; diff --git a/src/platform/x86_pc/clocks.cpp b/src/platform/x86_pc/clocks.cpp index 7955ca9d00..1200a717a3 100644 --- a/src/platform/x86_pc/clocks.cpp +++ b/src/platform/x86_pc/clocks.cpp @@ -18,6 +18,7 @@ #include "clocks.hpp" #include "../kvm/kvmclock.hpp" #include "cmos_clock.hpp" +#include "platform.hpp" #include #include #include @@ -40,7 +41,6 @@ static sysclock_t current_clock; namespace x86 { - void Clocks::init() { if (CPUID::kvm_feature(KVM_FEATURE_CLOCKSOURCE @@ -51,6 +51,7 @@ namespace x86 current_clock.system_time = {&KVM_clock::system_time}; current_clock.wall_time = {&KVM_clock::wall_clock}; current_clock.tsc_khz = {&KVM_clock::get_tsc_khz}; + x86::register_deactivation_function(KVM_clock::deactivate); INFO("x86", "KVM PV clocks initialized"); } } diff --git a/src/platform/x86_pc/platform.cpp b/src/platform/x86_pc/platform.cpp index 7eccd566c9..64c502db4a 100644 --- a/src/platform/x86_pc/platform.cpp +++ b/src/platform/x86_pc/platform.cpp @@ -44,6 +44,7 @@ static SMP::Array cpu_tables; namespace x86 { void initialize_cpu_tables_for_cpu(int cpu); + void register_deactivation_function(delegate); } @@ -125,6 +126,15 @@ void x86::initialize_cpu_tables_for_cpu(int cpu) #endif } +static std::vector> deactivate_funcs; +void x86::register_deactivation_function(delegate func) { + deactivate_funcs.push_back(std::move(func)); +} +void __arch_system_deactivate() +{ + for (auto& func : deactivate_funcs) func(); +} + void __arch_enable_legacy_irq(uint8_t irq) { x86::APIC::enable_irq(irq); diff --git a/src/platform/x86_pc/platform.hpp b/src/platform/x86_pc/platform.hpp new file mode 100644 index 0000000000..6a462b53a8 --- /dev/null +++ b/src/platform/x86_pc/platform.hpp @@ -0,0 +1,25 @@ +// This file is a part of the IncludeOS unikernel - www.includeos.org +// +// Copyright 2015 Oslo and Akershus University College of Applied Sciences +// and Alfred Bratterud +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include + +namespace x86 { + void register_deactivation_function(delegate func); +} + +extern void __platform_init(); diff --git a/test/lest_util/os_mock.cpp b/test/lest_util/os_mock.cpp index 3351ba1d55..d92fc11216 100644 --- a/test/lest_util/os_mock.cpp +++ b/test/lest_util/os_mock.cpp @@ -166,6 +166,7 @@ void __arch_reboot() {} void __arch_subscribe_irq(uint8_t) {} void __arch_enable_legacy_irq(uint8_t) {} void __arch_disable_legacy_irq(uint8_t) {} +void __arch_system_deactivate() {} delegate systime_override = [] () -> uint64_t { return 0; }; uint64_t __arch_system_time() noexcept { From 9f31b1f445658266bd70f80f291b0aa5b415e1f1 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Mon, 15 Oct 2018 19:14:02 +0200 Subject: [PATCH 720/723] nic: add defaults for sendq- and and rx buffer limits --- api/hw/nic.hpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/api/hw/nic.hpp b/api/hw/nic.hpp index 1117f5cb97..32e5ef7273 100644 --- a/api/hw/nic.hpp +++ b/api/hw/nic.hpp @@ -21,6 +21,9 @@ #include "mac_addr.hpp" #include +#define NIC_SENDQ_LIMIT_DEFAULT 4096 +#define NIC_BUFFER_LIMIT_DEFAULT 4096 + namespace hw { /** @@ -31,6 +34,12 @@ namespace hw { using upstream = delegate; using downstream = net::downstream_link; + /** Maximum number of packets allowed in dynamic send queue **/ + static constexpr uint32_t sendq_limit_default = NIC_SENDQ_LIMIT_DEFAULT; + + /** Maximum number of packets that can be created during RX-ring refill **/ + static constexpr uint32_t buffer_limit_default = NIC_BUFFER_LIMIT_DEFAULT; + enum class Proto {ETH, IEEE802111}; virtual Proto proto() const = 0; @@ -106,7 +115,7 @@ namespace hw { this->m_buffer_limit = new_limit; } uint32_t buffer_limit() const noexcept { return m_buffer_limit; } - + /** Set new sendq limit, where 0 means infinite **/ void set_sendq_limit(uint32_t new_limit) { this->m_sendq_limit = new_limit; @@ -156,8 +165,8 @@ namespace hw { private: int N; - uint32_t m_buffer_limit = 0; - uint32_t m_sendq_limit = 0; + uint32_t m_buffer_limit = buffer_limit_default; + uint32_t m_sendq_limit = sendq_limit_default; friend class Devices; }; From 47b80498dd572b5539743ba9c11349373f5784ae Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Mon, 15 Oct 2018 19:16:28 +0200 Subject: [PATCH 721/723] virtio: replace packet chain with deque, add stats --- src/drivers/virtionet.cpp | 94 ++++++++++++++++++++++++++++----------- src/drivers/virtionet.hpp | 14 +++++- test/stress/service.cpp | 22 ++++++--- 3 files changed, 97 insertions(+), 33 deletions(-) diff --git a/src/drivers/virtionet.cpp b/src/drivers/virtionet.cpp index 5f8d22a329..f6b8e0b3b7 100644 --- a/src/drivers/virtionet.cpp +++ b/src/drivers/virtionet.cpp @@ -65,10 +65,24 @@ VirtioNet::VirtioNet(hw::PCI_Device& d, const uint16_t /*mtu*/) Link(Link_protocol{{this, &VirtioNet::transmit}, mac()}), m_pcidev(d), bufstore_{VNET_TOT_BUFFERS(), 2048 /* half-page buffers */}, - packets_rx_{Statman::get().create(Stat::UINT64, - device_name() + ".packets_rx").get_uint64()}, - packets_tx_{Statman::get().create(Stat::UINT64, - device_name() + ".packets_tx").get_uint64()} + + stat_sendq_max_{Statman::get().create(Stat::UINT64, + device_name() + ".sendq_max").get_uint64()}, + stat_sendq_now_{Statman::get().create(Stat::UINT64, + device_name() + ".sendq_now").get_uint64()}, + stat_sendq_limit_dropped_{Statman::get().create(Stat::UINT64, + device_name() + ".sendq_dropped").get_uint64()}, + stat_rx_refill_dropped_{Statman::get().create(Stat::UINT64, + device_name() + ".rx_refill_dropped").get_uint64()}, + stat_bytes_rx_total_{Statman::get().create(Stat::UINT64, + device_name() + ".stat_rx_total_bytes").get_uint64()}, + stat_bytes_tx_total_{Statman::get().create(Stat::UINT64, + device_name() + ".stat_tx_total_bytes").get_uint64()}, + stat_packets_rx_total_{Statman::get().create(Stat::UINT64, + device_name() + ".stat_rx_total_packets").get_uint64()}, + stat_packets_tx_total_{Statman::get().create(Stat::UINT64, + device_name() + ".stat_tx_total_packets").get_uint64()} + { INFO("VirtioNet", "Driver initializing"); #undef VNET_TOT_BUFFERS @@ -216,7 +230,7 @@ void VirtioNet::msix_conf_handler() } void VirtioNet::msix_recv_handler() { - auto rx = packets_rx_; + auto rx = stat_packets_rx_total_; rx_q.disable_interrupts(); // handle incoming packets as long as bufstore has available buffers int max = 128; @@ -224,16 +238,24 @@ void VirtioNet::msix_recv_handler() { auto res = rx_q.dequeue(); VDBG_RX("[virtionet] Recv %u bytes\n", (uint32_t) res.size()); - Link::receive( recv_packet(res.data(), res.size()) ); - - // Requeue a new buffer - add_receive_buffer(bufstore().get_buffer()); + auto pckt = recv_packet(res.data(), res.size()); // Stat increase packets received - packets_rx_++; + stat_packets_rx_total_++; + stat_bytes_rx_total_ += pckt->size(); + + Link::receive(std::move(pckt)); + + // Requeue a new buffer unless threshold is reached + if (not Nic::buffers_still_available(bufstore().buffers_in_use())) + { + stat_rx_refill_dropped_++; + break; + } + add_receive_buffer(bufstore().get_buffer()); } rx_q.enable_interrupts(); - if (rx != packets_rx_) rx_q.kick(); + if (rx != stat_packets_rx_total_) rx_q.kick(); } void VirtioNet::msix_xmit_handler() { @@ -320,27 +342,47 @@ VirtioNet::create_packet(int link_offset) void VirtioNet::transmit(net::Packet_ptr pckt) { assert(pckt != nullptr); - if (transmit_queue == nullptr) - transmit_queue = std::move(pckt); - else - transmit_queue->chain(std::move(pckt)); + VDBG_TX("[virtionet] tx: Transmitting %#zu sized packet \n", + pckt->size()); + while (pckt != nullptr) { + if (not Nic::sendq_still_available(sendq.size())) { + stat_sendq_limit_dropped_++; + return; + } + auto tail = pckt->detach_tail(); + sendq.emplace_back(std::move(pckt)); + pckt = std::move(tail); + } + + // Update sendq stats + stat_sendq_now_ = sendq.size(); + if (sendq.size() > stat_sendq_max_) + stat_sendq_max_ = sendq.size(); + + auto tx = this->stat_packets_tx_total_; + + VDBG_TX("[virtionet] tx: packets in send queue %#zu\n", + sendq.size()); - auto tx = this->packets_tx_; // Transmit all we can directly - while (tx_q.num_free() > 1 and transmit_queue) + while (tx_q.num_free() > 1 and sendq.size() > 0) { - VDBG_TX("[virtionet] tx: %u tokens left in TX queue \n", + VDBG_TX("[virtionet] tx: %u tokens left in TX ring \n", tx_q.num_free()); - // next in line - auto next = transmit_queue->detach_tail(); - // explicitly release the data to prevent destructor being called - enqueue_tx(transmit_queue.release()); - transmit_queue = std::move(next); - // Stat increase packets transmitted - this->packets_tx_++; + + auto* next = sendq.front().release(); + sendq.pop_front(); + enqueue_tx(next); + + // Increase TX-stats + stat_packets_tx_total_++; + stat_bytes_tx_total_ += next->size(); + stat_packets_tx_total_++; } - if (tx != this->packets_tx_) { + VDBG_TX("[virtionet] tx: packet enqueued\n"); + + if (tx != this->stat_packets_tx_total_) { #ifdef NO_DEFERRED_KICK tx_q.kick(); #else diff --git a/src/drivers/virtionet.hpp b/src/drivers/virtionet.hpp index 51721b6144..7a4c59c1d6 100644 --- a/src/drivers/virtionet.hpp +++ b/src/drivers/virtionet.hpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -236,8 +237,17 @@ class VirtioNet : Virtio, public net::Link_layer { net::BufferStore bufstore_; /** Stats */ - uint64_t& packets_rx_; - uint64_t& packets_tx_; + uint64_t& stat_sendq_max_; + uint64_t& stat_sendq_now_; + uint64_t& stat_sendq_limit_dropped_; + uint64_t& stat_rx_refill_dropped_; + uint64_t& stat_bytes_rx_total_; + uint64_t& stat_bytes_tx_total_; + uint64_t& stat_packets_rx_total_; + uint64_t& stat_packets_tx_total_; + + std::deque sendq{}; + }; #endif diff --git a/test/stress/service.cpp b/test/stress/service.cpp index 1e603bed97..ed948a9c3a 100644 --- a/test/stress/service.cpp +++ b/test/stress/service.cpp @@ -21,6 +21,7 @@ #include // rand() #include #include +#include using namespace std::chrono; @@ -159,16 +160,27 @@ void Service::start(const std::string&) net::UDP::port_t port_mem = 4243; auto& conn_mem = inet.udp().bind(port_mem); -/* - Timers::periodic(10s, 10s, - [] (Timers::id_t) { - printf(" TCP STATUS:\n%s \n", inet.tcp().status().c_str()); + Timers::periodic(1s, 10s, + [] (Timers::id_t) { auto memuse = OS::heap_usage(); printf("Current memory usage: %i b, (%f MB) \n", memuse, float(memuse) / 1000000); printf("Recv: %llu Sent: %llu\n", TCP_BYTES_RECV, TCP_BYTES_SENT); + printf("eth0.sendq_max: %zu, eth0.sendq_now: %zu" + "eth0.stat_rx_total_packets: %zu, eth0.stat_rx_total_packets: %zu, " + "eth0.stat_rx_total_bytes: %zu, eth0.stat_tx_total_bytes: %zu, " + "eth0.sendq_dropped: %zu, eth0.rx_refill_dropped: %zu \n", + Statman::get().get_by_name("eth0.sendq_max").get_uint64(), + Statman::get().get_by_name("eth0.sendq_now").get_uint64(), + Statman::get().get_by_name("eth0.stat_rx_total_packets").get_uint64(), + Statman::get().get_by_name("eth0.stat_tx_total_packets").get_uint64(), + Statman::get().get_by_name("eth0.stat_rx_total_bytes").get_uint64(), + Statman::get().get_by_name("eth0.stat_tx_total_bytes").get_uint64(), + Statman::get().get_by_name("eth0.sendq_dropped").get_uint64(), + Statman::get().get_by_name("eth0.rx_refill_dropped").get_uint64() + ); }); -*/ + server_mem.on_connect([] (net::tcp::Connection_ptr conn) { conn->on_read(1024, [conn](net::tcp::buffer_t buf) { TCP_BYTES_RECV += buf->size(); From 052682f87a323fbe123de32a293fdcbac9096ab1 Mon Sep 17 00:00:00 2001 From: Alfred Bratterud Date: Fri, 2 Nov 2018 10:49:01 +0100 Subject: [PATCH 722/723] virtionet: remove old transmit_queue. Fixes transmission bug. --- src/drivers/virtionet.cpp | 8 +++----- src/drivers/virtionet.hpp | 1 - 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/drivers/virtionet.cpp b/src/drivers/virtionet.cpp index f6b8e0b3b7..4437f6fe03 100644 --- a/src/drivers/virtionet.cpp +++ b/src/drivers/virtionet.cpp @@ -279,14 +279,12 @@ void VirtioNet::msix_xmit_handler() VDBG_TX("[virtionet] %d transmitted\n", dequeued_tx); // transmit as much as possible from the buffer - if (transmit_queue != nullptr) { - //auto tx = packets_tx_; - transmit(std::move(transmit_queue)); - //printf("[virtionet] %ld transmit queue\n", packets_tx_ - tx); + if (! sendq.empty()) { + transmit(nullptr); } // If we now emptied the buffer, offer packets to stack - if (transmit_queue == nullptr && tx_q.num_free() > 1) { + if (sendq.empty() && tx_q.num_free() > 1) { transmit_queue_available_event(tx_q.num_free() / 2); } } diff --git a/src/drivers/virtionet.hpp b/src/drivers/virtionet.hpp index 7a4c59c1d6..cdded90431 100644 --- a/src/drivers/virtionet.hpp +++ b/src/drivers/virtionet.hpp @@ -233,7 +233,6 @@ class VirtioNet : Virtio, public net::Link_layer { bool deferred_kick = false; static void handle_deferred_devices(); - net::Packet_ptr transmit_queue = nullptr; net::BufferStore bufstore_; /** Stats */ From e34faccb131507986f8d28930e9d46c3d9ab65df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alf-Andr=C3=A9=20Walla?= Date: Mon, 22 Oct 2018 14:15:38 +0200 Subject: [PATCH 723/723] virtionet: Fix a few issues with sendq --- src/drivers/virtionet.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/src/drivers/virtionet.cpp b/src/drivers/virtionet.cpp index 4437f6fe03..42351947c0 100644 --- a/src/drivers/virtionet.cpp +++ b/src/drivers/virtionet.cpp @@ -265,9 +265,9 @@ void VirtioNet::msix_xmit_handler() while (tx_q.new_incoming()) { auto res = tx_q.dequeue(); - // get packet offset, and call destructor - auto* packet = (net::Packet*) (res.data() - sizeof(net::Packet)); - delete packet; // call deleter on Packet to release it + assert(res.data() != nullptr); + // get packet offset, and call placement Packet deleter directly + net::Packet::operator delete(res.data() - sizeof(net::Packet)); dequeued_tx++; } tx_q.enable_interrupts(); @@ -339,14 +339,13 @@ VirtioNet::create_packet(int link_offset) void VirtioNet::transmit(net::Packet_ptr pckt) { - assert(pckt != nullptr); - VDBG_TX("[virtionet] tx: Transmitting %#zu sized packet \n", - pckt->size()); while (pckt != nullptr) { if (not Nic::sendq_still_available(sendq.size())) { - stat_sendq_limit_dropped_++; - return; + stat_sendq_limit_dropped_ += pckt->chain_length(); + break; } + VDBG_TX("[virtionet] tx: Transmitting %#zu sized packet \n", + pckt->size()); auto tail = pckt->detach_tail(); sendq.emplace_back(std::move(pckt)); pckt = std::move(tail); @@ -363,7 +362,7 @@ void VirtioNet::transmit(net::Packet_ptr pckt) sendq.size()); // Transmit all we can directly - while (tx_q.num_free() > 1 and sendq.size() > 0) + while (tx_q.num_free() > 1 and !sendq.empty()) { VDBG_TX("[virtionet] tx: %u tokens left in TX ring \n", tx_q.num_free());