Differences between revisions 1 and 3 (spanning 2 versions)

neru has modest hardware specs but I'm not planning to use her as a router. She's a small, low-power appliance that can run containers and takes a USB stick for extra storage.

neru is a hexS 2025 model:

Homepage: https://mikrotik.com/product/hex_s_2025
Product code: E60iUGS

Contents

Features and uses
Hardware
Config
1. Config dump
Original setup exploration

Features and uses

5x 1G RJ45 ports
1x 2.5G SFP port
PoE powered
arm CPU, meaning we can run containers

Hardware

CPU: EN7562CT
Switch chip: EN7523 (https://help.mikrotik.com/docs/spaces/ROS/pages/15302988/Switch+Chip+Features)

A surprisingly decent switch chip for such a cheap device, it does hardware STP and VLAN filtering sensibly.

Config

Notes from when I set her up as a PXE server, running TFTP (native) and HTTP (nginx in a container), serving files from the USB stick.

Served up here: http://neru.thighhighs.top:8080/

Config dump

Taken from a /export on the CLI.

It might look a bit unusual because there's no default bridge, we're only using ether1 as the WAN port, and NATing traffic to/from the container NIC running on there.

# This is for the container NICs to attach to. Not sure if we need it, I dunno if host-mode networking would obviate it.
/interface bridge
add name=containers priority=0xd000 protocol-mode=mstp vlan-filtering=yes

# Virtual NIC that the container will own. We give it an IP address that we'll NAT from/to
/interface veth
add address=172.25.0.2/24 container-mac-address=2E:9E:C1:16:16:92 dhcp=no gateway=172.25.0.1 gateway6="" mac-address=2E:9E:C1:16:16:91 name=nginx

# These VLANs live directly on ether1 because it's all that matters, we're not using the default bridge.
# Probably should remove the DN42 VLAN, as we don't need to provide TFTP/PXE on that subnet.
/interface vlan
add interface=ether1 name=VLAN10_GENERAL vlan-id=10
add interface=ether1 name=VLAN11_MGMT vlan-id=11
add interface=ether1 name=VLAN42_DN42_LAN vlan-id=42

/interface list
add comment=defconf name=WAN
add comment=defconf name=LAN

/user group
add name=mikrodash policy=read,test,api,!local,!telnet,!ssh,!ftp,!reboot,!write,!policy,!winbox,!password,!web,!sniff,!sensitive,!romon,!rest-api

/container
add domain-name=thighhighs.top hostname=neru interface=nginx layer-dir="" logging=yes memory-high=128.0MiB mount=\
    /usb1p1/srv:/usr/share/nginx/html:ro,/usb1p1/nginx-conf.d:/etc/nginx/conf.d:ro name=nginx remote-image=arm32v5/nginx:trixie root-dir=\
    /usb1p1/container-roots/nginx start-on-boot=yes
/container config
set memory-high=256.0MiB registry-url=https://registry-1.docker.io tmpdir=/usb1p1/tmp

/disk
set usb1 media-interface=*8 media-sharing=yes smb-sharing=yes
add media-interface=*8 media-sharing=yes mount-point-template=usb1p1 parent=usb1 partition-number=1 partition-offset=16384 partition-size=61524131840 smb-sharing=\
    yes type=partition

/disk settings
set auto-media-interface=*8

/interface bridge port
add bridge=containers interface=nginx

/ip neighbor discovery-settings
set discover-interface-list=all lldp-mac-phy-config=yes lldp-max-frame-size=yes lldp-vlan-info=yes

/interface list member
add comment=defconf interface=ether1 list=WAN
add interface=VLAN11_MGMT list=LAN
add interface=VLAN42_DN42_LAN list=LAN
add interface=VLAN10_GENERAL list=LAN

/ip address
add address=192.168.11.9/24 interface=VLAN11_MGMT network=192.168.11.0
add address=172.25.0.1/24 interface=containers network=172.25.0.0

# Get our "public" IP address via DHCP
/ip dhcp-client
add comment=defconf interface=VLAN10_GENERAL name=client1

# Can be queried as a nameserver, probably to serve the containers (not that they would need to, I think)
/ip dns
set allow-remote-requests=yes

###
### WIP
### 
/ip firewall filter
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: accept ICMP" protocol=icmp
add action=accept chain=input comment="defconf: accept to local loopback (for CAPsMAN)" dst-address=127.0.0.1
add action=accept chain=input comment="udp/5678 Neighbour Discovery" dst-address=255.255.255.255 dst-port=5678 log-prefix="udp/5678 Neighbour Discovery" protocol=\
    udp
add action=accept chain=input comment="winbox from WAN" dst-port=8291 in-interface=ether1 protocol=tcp
add action=accept chain=input comment="BGP from MGMT" dst-port=179 in-interface=VLAN11_MGMT protocol=tcp
add action=accept chain=input comment="wireguard from hk1.kioubit" dst-port=23915 protocol=udp
add action=drop chain=input comment="drop Dropbox LAN discovery proto" dst-port=17500 protocol=udp src-port=17500
add action=drop chain=input comment="drop Google Home IOT noise" dst-port=9999 protocol=udp src-port=9999
add action=drop chain=input comment="drop Google Home IOT noise 2" dst-address=255.255.255.255 dst-port=9478 protocol=udp src-port=9478
add action=drop chain=input comment="drop SSDP noise" dst-address=255.255.255.255 dst-port=1900 protocol=udp
add action=drop chain=input comment="defconf: drop all not coming from LAN" in-interface-list=!LAN log-prefix="drop non-LAN"
add action=accept chain=forward comment="defconf: accept in ipsec policy" ipsec-policy=in,ipsec
add action=accept chain=forward comment="defconf: accept out ipsec policy" ipsec-policy=out,ipsec
add action=fasttrack-connection chain=forward comment="defconf: fasttrack" connection-state=established,related
add action=accept chain=forward comment="defconf: accept established,related, untracked" connection-state=established,related,untracked
add action=accept chain=forward comment="local nginx (post-dstnat)" disabled=yes dst-address=172.25.0.2 dst-port=80 in-interface=containers protocol=tcp
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=drop chain=forward comment="defconf: drop all from WAN not DSTNATed" connection-nat-state=!dstnat connection-state=new in-interface-list=WAN
/ip firewall nat
add action=masquerade chain=srcnat comment="defconf: masquerade" ipsec-policy=out,none out-interface-list=WAN
add action=masquerade chain=srcnat src-address=172.25.0.0/24
add action=dst-nat chain=dstnat dst-address=192.168.1.25 dst-port=8080 protocol=tcp to-addresses=172.25.0.2 to-ports=80
/ip tftp
add read-only=no real-filename=usb1p1/srv/ req-filename=.*
/ip tftp settings
set max-block-size=8192
/ipv6 address
add address=fdfd:2501:2762:11::7 interface=VLAN11_MGMT
/ipv6 firewall address-list
add address=::/128 comment="defconf: unspecified address" list=bad_ipv6
add address=::1/128 comment="defconf: lo" list=bad_ipv6
add address=fec0::/10 comment="defconf: site-local" list=bad_ipv6
add address=::ffff:0.0.0.0/96 comment="defconf: ipv4-mapped" list=bad_ipv6
add address=::/96 comment="defconf: ipv4 compat" list=bad_ipv6
add address=100::/64 comment="defconf: discard only " list=bad_ipv6
add address=2001:db8::/32 comment="defconf: documentation" list=bad_ipv6
add address=2001:10::/28 comment="defconf: ORCHID" list=bad_ipv6
add address=3ffe::/16 comment="defconf: 6bone" list=bad_ipv6
/ipv6 firewall filter
add action=accept chain=input comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=input comment="defconf: drop invalid" connection-state=invalid
add action=accept chain=input comment="defconf: accept ICMPv6" protocol=icmpv6
add action=accept chain=input comment="iBGP in from MGMT" dst-port=179 in-interface=VLAN11_MGMT protocol=tcp
add action=accept chain=input comment="defconf: accept UDP traceroute" dst-port=33434-33534 protocol=udp
add action=accept chain=input comment="defconf: accept DHCPv6-Client prefix delegation." dst-port=546 protocol=udp src-address=fe80::/10
add action=accept chain=input comment="defconf: accept IKE" dst-port=500,4500 protocol=udp
add action=accept chain=input comment="defconf: accept ipsec AH" protocol=ipsec-ah
add action=accept chain=input comment="defconf: accept ipsec ESP" protocol=ipsec-esp
add action=accept chain=input comment="defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=input comment="defconf: drop everything else not coming from LAN" in-interface-list=!LAN
add action=fasttrack-connection chain=forward comment="defconf: fasttrack6" connection-state=established,related
add action=accept chain=forward comment="defconf: accept established,related,untracked" connection-state=established,related,untracked
add action=drop chain=forward comment="defconf: drop invalid" connection-state=invalid
add action=drop chain=forward comment="defconf: drop packets with bad src ipv6" src-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: drop packets with bad dst ipv6" dst-address-list=bad_ipv6
add action=drop chain=forward comment="defconf: rfc4890 drop hop-limit=1" hop-limit=equal:1 protocol=icmpv6
add action=accept chain=forward comment="defconf: accept ICMPv6" protocol=icmpv6
add action=accept chain=forward comment="defconf: accept HIP" protocol=139
add action=accept chain=forward comment="defconf: accept IKE" dst-port=500,4500 protocol=udp
add action=accept chain=forward comment="defconf: accept ipsec AH" protocol=ipsec-ah
add action=accept chain=forward comment="defconf: accept ipsec ESP" protocol=ipsec-esp
add action=accept chain=forward comment="defconf: accept all that matches ipsec policy" ipsec-policy=in,ipsec
add action=drop chain=forward comment="defconf: drop everything else not coming from LAN" in-interface-list=!LAN
/snmp
set contact=Furinkan enabled=yes location="Conferta Ave"
/system clock
set time-zone-name=Australia/Sydney
/system identity
set name=neru
/system ntp client
set enabled=yes
/system ntp client servers
add address=pool.ntp.org
add address=au.pool.ntp.org
/system routerboard settings
set silent-boot=yes
/tool mac-server
set allowed-interface-list=LAN
/tool mac-server mac-winbox
set allowed-interface-list=LAN
/tool sniffer
set filter-operator-between-entries=and filter-stream=yes memory-limit=1000KiB streaming-enabled=yes streaming-server=192.168.1.64

Original setup exploration

I have aris, she's an original hex S with microSD card slot and USB. She has an mmips CPU, which means no container support

But I have neru, she's a hex S 2025 model, no microSD but with USB. I'd prefer microSD because I have spares, but beggars can't be choosers. I really don't want stuff sticking out, so it'll have to be a micro-profile USB stick.

I've already messed with this before, I've enabled container mode (https://help.mikrotik.com/docs/spaces/ROS/pages/93749258/Device-mode#Devicemode-Enablingdevice-modefeature) and installed the container package.

pop in the usb stick (Sandisk Ultra Fit 64gb USB 3.2 gen1 5Gbps), it already has a partition and FS. Set the mountpoint and format it.

/disk/set usb1-part1 mount-point-template=usb1p1
/disk/settings/set auto-media-sharing=no auto-smb-sharing=no

/disk/format usb1-part1 file-system=ext4 label=neru-data

Follow the network setup outline, I've picked an IP range that shouldn't conflict with any of my stuff. The suggested docker network settings are fine in most cases though: https://help.mikrotik.com/docs/spaces/ROS/pages/84901929/Container#Container-RunningPi-hole

/interface/bridge/add name=containers priority=0xD000 protocol-mode=mstp vlan-filtering=yes
/ip/address/add address=172.25.0.1/24 interface=containers

/interface/veth/add name=netbooting address=172.25.0.2/24 gateway=172.25.0.1
/interface/bridge/port/add bridge=containers interface=netbooting

/ip/firewall/nat/add chain=srcnat action=masquerade src-address=172.25.0.0/24

Setup container stuff

/file/add type=directory name=usb1p1/containers
/file/add type=directory name=usb1p1/srv
/file/add type=directory name=usb1p1/tmp

/container/config/set tmpdir=usb1p1/tmp
/container/config/set registry-url=https://registry-1.docker.io
/container/config/set memory-high=256MiB

/container/add auto-restart-interval=1m domain-name=thighhighs.top hostname=neru interface=netbooting logging=yes memory-high=128.0MiB mount=/usb1p1/srv:/var/www:ro name=caddy remote-image=library/caddy root-dir=/usb1p1/container-roots/caddy start-on-boot=yes workdir=/srv

The container dies immediately after start, log shows "exited with signal 4", which is a SIGILL. I'm almost certain this is an architecture limitation. Sure enough...

> For devices with EN7562CT CPU like the hEX Refresh, only arm32v5 container images are supported

I bet it's more than just that. I need a caddy image that supports v5 specifically. It's probably pulled a v7 or v8 image.

Okay caddy is too much fucking work. I'd have to build it myself on a different machine, and the Dockerfile for it only uses alpine, which isn't supported on arm32v5. I'll just use nginx, I only need static fileserving anyway so it's all moot.

https://hub.docker.com/r/arm32v5/nginx

/container/add name=nginx remote-image=arm32v5/nginx:trixie check-certificate=yes interface=netbooting mount=/usb1p1/srv:/usr/share/nginx/html:ro hostname=neru domain-name=thighhighs.top memory-high=128.0MiB root-dir=/usb1p1/container-roots/nginx start-on-boot=yes logging=yes

/container/start nginx

/ip/firewall/nat/add chain=dstnat dst-address=192.168.1.25 dst-port=8080 protocol=tcp action=dst-nat to-addresses=172.25.0.2 to-ports=80

Configure it:

root@abc3def4c047:/etc/nginx/conf.d# cat default.conf
server {
    listen       80;
    server_name  localhost;

    #access_log  /var/log/nginx/host.access.log  main;

    location / {
        root   /usr/share/nginx/html;
        index  index.html index.htm;
    }

    #error_page  404              /404.html;

    # redirect server error pages to the static page /50x.html
    #
    error_page   500 502 503 504  /50x.html;
    location = /50x.html {
        root   /usr/share/nginx/html;
    }

    # proxy the PHP scripts to Apache listening on 127.0.0.1:80
    #
    #location ~ \.php$ {
    #    proxy_pass   http://127.0.0.1;
    #}

    # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
    #
    #location ~ \.php$ {
    #    root           html;
    #    fastcgi_pass   127.0.0.1:9000;
    #    fastcgi_index  index.php;
    #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
    #    include        fastcgi_params;
    #}

    # deny access to .htaccess files, if Apache's document root
    # concurs with nginx's one
    #
    #location ~ /\.ht {
    #    deny  all;
    #}
}

Useful(?) links

Navigation

Features and uses

Hardware

Config

Config dump

Original setup exploration