\documentclass{report}
\usepackage{amsthm,amssymb,amsmath}
\usepackage{systeme}
\usepackage{xepersian}
\makeatletter
\@ifundefined{Umathcode}{\let\Umathcode\XeTeXmathcode}{}
\@ifundefined{Umathchardef}{\let\Umathchardef\XeTeXmathchardef}{}
\makeatother




\begin{document}

$$\sysdelim..\systeme{2a+3b=5,3ab=
8}$$
$$\sysdelim.|\systeme{2a+3b=5,3ab=
8}$$
\end{document}